Esempio n. 1
0
    def __init__(self, xml=None):
        if type(xml) is NoneType:
            self._metricComparator = None
            self._metricName = None
            self._result = None
            self._parameters = []
        elif type(xml) is StringType or iselement(xml):
            if type(xml) is StringType:
                root = ElementTree.fromstring(xml)
            elif iselement(xml):
                root = xml

            if root.tag != "value":
                raise PerfRepoException("Invalid xml.")

            self._metricComparator = root.get("metricComparator")
            self._metricName = root.get("metricName")
            self._result = float(root.get("result"))

            self._parameters = []
            for param in root.find("parameters"):
                if param.tag != "parameter":
                    continue
                self._parameters.append((param.get("name"), param.get("value")))
        else:
            raise PerfRepoException("Parameter xml must be"\
                                    " a string, an Element or None")
Esempio n. 2
0
    def __init__(self, xml=None):
        self._user = None
        if type(xml) is NoneType:
            self._id = None
            self._name = None
            self._type = None
            self._properties = {}
        elif type(xml) is StringType or iselement(xml):
            if type(xml) is StringType:
                root = ElementTree.fromstring(xml)
            elif iselement(xml):
                root = xml

            if root.tag != "report":
                raise PerfRepoException("Invalid xml.")

            self._id = root.get("id")
            self._name = root.get("name")
            self._type = root.get("type")
            self._properties = {}
            for entry in root.find("properties"):
                if entry.tag != "entry":
                    continue
                key_tag = entry.find("key")
                value_tag = entry.find("value")
                tmp_dict = dot_to_dict(value_tag.get("name"),
                                       value_tag.get("value"))
                recursive_dict_update(self._properties, tmp_dict)
        else:
            raise PerfRepoException("Parameter xml must be"\
                                    " a string, an Element or None")
Esempio n. 3
0
    def __init__(self, xml=None):
        if type(xml) is NoneType:
            self._id = None
            self._name = None
            self._uid = None
            self._description = None
            self._groupid = None
            self._metrics = []
        elif type(xml) is StringType or iselement(xml):
            if type(xml) is StringType:
                root = ElementTree.fromstring(xml)
            elif iselement(xml):
                root = xml

            if root.tag != "test":
                raise PerfRepoException("Invalid xml.")

            self._id = root.get("id")
            self._name = root.get("name")
            self._uid = root.get("uid")
            self._groupid = root.get("groupId")
            self._description = root.find("description").text
            self._metrics = []
            for metric in root.find("metrics"):
                if metric.tag != "metric":
                    continue
                self._metrics.append(PerfRepoMetric(metric))
        else:
            raise PerfRepoException("Parameter xml must be"\
                                    " a string, an Element or None")
Esempio n. 4
0
def parserMascotXMLAndWriteToFile(inputFileName, outputFileName):
    try:
        with open(outputFileName, 'w') as fw:
            fw.write('\t'.join(COLNAMES) + '\n')
            tree = ET.ElementTree(file=inputFileName)
            for elem in tree.iterfind(tag_queries_query_q_peptide):
                query_num = elem.attrib['query']
                query_rank = elem.attrib['rank']
                # queries -> query -> q_peptide
                tag_pep_seq = TAG_PREFIX + 'pep_seq'
                tag_pep_score = TAG_PREFIX + 'pep_score'
                tag_pep_var_mod = TAG_PREFIX + 'pep_var_mod'  # Phospho
                tag_pep_var_mod_pos = TAG_PREFIX + 'pep_var_mod_pos'
                tag_pep_var_mod_conf = TAG_PREFIX + 'pep_var_mod_conf'

                e_tag_pep_seq = elem.find(tag_pep_seq)
                e_tag_pep_score = elem.find(tag_pep_score)
                e_tag_pep_var_mod = elem.find(tag_pep_var_mod)
                e_tag_pep_var_mod_pos = elem.find(tag_pep_var_mod_pos)
                e_tag_pep_var_mod_conf = elem.find(tag_pep_var_mod_conf)

                pep_seq = e_tag_pep_seq.text
                pep_score = e_tag_pep_score.text

                if iselement(e_tag_pep_var_mod
                             ) and e_tag_pep_var_mod.text is not None:
                    pep_var_mod = e_tag_pep_var_mod.text
                else:
                    pep_var_mod = ''

                if iselement(e_tag_pep_var_mod_pos
                             ) and e_tag_pep_var_mod_pos.text is not None:
                    pep_var_mod_pos = e_tag_pep_var_mod_pos.text
                else:
                    pep_var_mod_pos = ''

                if iselement(e_tag_pep_var_mod_conf
                             ) and e_tag_pep_var_mod_conf.text is not None:
                    pep_var_mod_conf = e_tag_pep_var_mod_conf.text
                else:
                    pep_var_mod_conf = ''

                if pep_var_mod_pos != '':
                    pep_seq = formatting_pep_seq(pep_seq, pep_var_mod_pos)

                lst = [
                    pep_seq, query_num, query_rank, pep_score, pep_var_mod,
                    pep_var_mod_pos, pep_var_mod_conf
                ]
                lst_str = '\t'.join(lst)
                fw.write(lst_str + '\n')

    except RuntimeError:
        return False
    else:
        return True
Esempio n. 5
0
def download_main_file():
    """
    The main function to handle all
    :return:
    """
    url = "https://registers.esma.europa.eu/solr/esma_registers_firds_files/select"
    params = {
        "q": "*",
        "fq":
        "publication_date:[2021-01-17T00:00:00Z TO 2021-01-19T23:59:59Z]",
        "wt": "xml",
        "indent": "true",
        "start": "0",
        "rows": "100"
    }
    try:
        response = requests.get(url=url, params=params)
        logger.info(f"Successfully request is connected {url}")
    except ConnectionError:
        logger.error("Not able to connect to internet", exc_info=True)
        return False
    if response.status_code == 200:
        tree = ElementTree.fromstring(response.content.decode())
        all_docs = tree.find('result').findall("doc")
        result_doc = None
        for doc in all_docs:
            if doc.find("str[@name='file_type']").text == 'DLTINS':
                result_doc = doc
                break
        if not iselement(result_doc):
            logger.info("Did not get any file type as `DLTINS`")
            return False
        download_doc = result_doc.find("str[@name='download_link']")
        if iselement(download_doc):
            download_link = download_doc.text
            xml_tree_root = get_xml_tree_from_zip(download_link)
            if not iselement(xml_tree_root):
                logger.info("Zip is not converted to xml tree")
                return False
            csv_file = xml_to_csv(xml_tree_root)
            if not csv_file:
                return False
            # TODO: uncomment below line to upload s3 bucket
            # status = upload_to_s3(csv_file)
            # logger.info("All task is done")
            # return status
        else:
            logger.warning("Did not found any download link to download zip")
            return False
    else:
        logger.error(
            f"Request is not proper. got status code ={response.status_code}, {response.content}",
            exc_info=True)
        return False
Esempio n. 6
0
    def __init__(self, xml=None):
        if type(xml) is NoneType:
            self._id = None
            self._name = None
            self._started = datetime.datetime.utcnow().isoformat()
            self._testId = None
            self._testUid = None
            self._comment = ""

            self._values = []
            self._tags = []
            self._parameters = []
        elif type(xml) is StringType or iselement(xml):
            if type(xml) is StringType:
                root = ElementTree.fromstring(xml)
            elif iselement(xml):
                root = xml

            if root.tag != "testExecution":
                raise PerfRepoException("Invalid xml.")

            self._id = root.get("id")
            self._name = root.get("name")
            self._started = root.get("started")
            self._testId = root.get("testId")
            self._testUid = root.get("testUid")
            self._comment = root.find("comment").text

            self._values = []
            for value in root.find("values"):
                if value.tag != "value":
                    continue
                self._values.append(PerfRepoValue(value))

            self._tags = []
            for tag in root.find("tags"):
                if tag.tag != "tag":
                    continue
                self._tags.append(tag.get("name"))

            self._parameters = []
            for param in root.find("parameters"):
                if param.tag != "parameter":
                    continue
                self._parameters.append((param.get("name"), param.get("value")))
        else:
            raise PerfRepoException("Parameter xml must be"\
                                    " a string, an Element or None")
Esempio n. 7
0
    def __init__(self, xml=None):
        self._report_id = None
        self._access_type = None
        self._access_level = None
        self._user_id = None
        self._group_id = None
        if type(xml) is NoneType:
            pass
        elif type(xml) is StringType or iselement(xml):
            if type(xml) is StringType:
                root = ElementTree.fromstring(xml)
            else:
                root = xml
            if root.tag != "report-permission":
                raise PerfRepoException("Invalid xml.")

            self._report_id = root.get("report-id").text
            self._access_type = root.get("access-type").text
            self._access_level = root.get("access-level").text

            try:
                self._user_id = root.get("user-id").text
            except:
                self._user_id = None
            try:
                self._group_id = root.get("group-id").text
            except:
                self._group_id = None

        else:
            raise PerfRepoException("Parameter xml must be"\
                                    " a string, an Element or None")
    def __init__(self, xml=None):
        self._ids = None
        self._testUid = None
        self._testname = None
        self._tags = []
        self._parameters = []

        if type(xml) is StringType or iselement(xml):
            if type(xml) is StringType:
                root = ElementTree.fromstring(xml)
            else:
                root = xml
            if root.tag != "test-execution-search":
                raise PerfRepoException("Invalid xml.")

            elem_ids = root.find("ids")
            if elem_ids is not None:
                ids = []
                for id in elem_ids:
                    ids.append(id.text)
                self.set_ids(ids)

            self._testname = root.get("test-name")
            self._testUid = root.get("test-uid")
            elem_tags = root.find("tags")
            if elem_tags is not None and elem_tags.text is not None:
                tags = root.find("tags").text
                self._tags = tags.split()

            params = root.find("parameters")
            for param in params.findall("parameter"):
                pname = param.find("name")
                pvalue = param.find("value")
                self.add_parameter(pname.text, pvalue.text)
def xmleval(source_tree, target_tree):
    """
    Searches for the target_tree in the XML source tree
    Returns the number of occurance and location of each occurrence.
    target_string: the string of query element tree
    goal_list: a list of target_tree found in source tree.
    output_list is the list of output:
    first element in the list denotes the number of occurance
    other elements denote the location(id) of each occurance
    source_tree2 : created by inserting source tree in an empty element tree.
                   the reason to do so is to include the top layer of source tree in the searching.
    Using element.set() and element.get() to set and get attribute key on the element to value,
                   which reduce algorithm complexity compared to previous nested searching for loop
    """
  
    Ocurrence_counter = 0
    output_list = []
    output_list.append(Ocurrence_counter)
    
    Start_String = ".//"    # Selects all subelements on all level beneath the current element
    target_string = get_Target_String(target_tree,Start_String)
    

    tag_list = source_tree.iter()
    for (location,tag) in enumerate (tag_list,1):
        tag.set('#', str(location))
    
    '''
    # --------------previous indexing methods, retired------------------    
    index_tag = 1
    index_list = []
    for entry in tag_list:
        index_list.append([index_tag, entry])        
        index_tag += 1
    # ------------------------------------------------------------------
    '''
    source_tree2 = Element("")
    source_tree2.insert(0,source_tree)
    assert(iselement(source_tree2) == True)
    goal_list = (source_tree2.findall(target_string))   
    assert(type(goal_list) is list)
    
    for entry in goal_list:
        output_list.append(int(entry.get('#')))
        Ocurrence_counter +=1
    '''       
    # --------------previous indexing methods, retired------------------       
    for x in goal_list:
        for y in index_list:
            if x == y[1]:
                assert(type(y[0] is int))
                output_list.append(y[0])
                Ocurrence_counter +=1
    # ------------------------------------------------------------------
    '''          
    output_list[0] = Ocurrence_counter   
    assert(type(output_list) is list)
    return output_list 
Esempio n. 10
0
def coerce_response(response):
    if isinstance(response, basestring):
        response = HTTPResponse(HTTPPreamble(headers={'Content-Type': 'text/html'}), body=response)
    elif isinstance(response, dict):
        response = HTTPResponse(HTTPPreamble(headers={'Content-Type': 'application/json'}), body=json.dumps(response))
    elif iselement(response):
        xml = StringIO()
        ElementTree(response).write(xml)
        response = HTTPResponse(HTTPPreamble(headers={'Content-Type': 'application/xml'}), body=xml.getvalue())
    return response
Esempio n. 11
0
    def parse_cards(self, chassis_element):
        if not iselement(chassis_element):
            raise Exception("%s is not an element." % chassis_element)

        card_list = list(chassis_element)
        card_info = {}
        for card in card_list:
            cardName = card.attrib['Value']
            card_info[cardName] = {"aid": card.attrib['aid'], "Pon": card.attrib['PON']}

        return card_info
Esempio n. 12
0
 def parse_chassis(self, ne_element):
     if not iselement(ne_element):
         raise Exception("%s is not an element." % ne_element)
     chassis_list = list(ne_element)
     chassisInfo = {}
     for chassis in chassis_list:
         chassisName = chassis.attrib['Value']
         chs_element = list(chassis)[0]
         cardInfo = self.parse_cards(chs_element)
         chassisInfo[chassisName] = {'shelfId': chassis.attrib['aid'], 'chassisNo': chassis.attrib['CHASSERNO'],
                                     'chsCategory': chs_element.tag, 'cardInfo': cardInfo}
     return chassisInfo
Esempio n. 13
0
def writeTableColumn(data,tableName):
    #Get new data to write to the config as a table column
    try:
        tree, root=parseRootElement()
        for elem in root.findall('table'):
            if elem.get('name')==tableName:
                parent=Element(tableName)
                if iselement(parent):
                    node=elem.find("columns")
                    SubElement(node,"column", data)
        tree.write(xml_doc)
    except:
        pass
Esempio n. 14
0
    def raise_for_response(self, response, valid_response):
        if response.status != valid_response:
            error_msg = response.body
            try:
                body = parse_xml(error_msg)
                if iselement(body):
                    error_msg = parse_error(body)
            except XMLParseError:
                pass

            values = (response.error, error_msg, response.status)
            message = 'Message: %s, Body: %s, Status code: %s' % (values)
            raise LibcloudError(message, driver=self)
Esempio n. 15
0
    def __init__(self, xml=None):
        if type(xml) is NoneType:
            self._id = None
            self._name = None
            self._description = None
            self._comparator = None
        elif type(xml) is StringType or iselement(xml):
            if type(xml) is StringType:
                root = ElementTree.fromstring(xml)
            elif iselement(xml):
                root = xml

            if root.tag != "metric":
                raise PerfRepoException("Invalid xml.")

            self._id = root.get("id")
            self._name = root.get("name")
            self._comparator = root.get("comparator")
            self._description = root.find("description").text
        else:
            raise PerfRepoException("Parameter xml must be"\
                                    " a string, an Element or None")
Esempio n. 16
0
def writeTableColumn(data, profile, category, tableName, tableNode):
    #Get new data to write to the config as a table column
    #node=None
    tree, root = parseRootElement()
    filters = (".//*[@name='%s']/%s") % (profile, category)
    for elem in root.findall(filters):
        if elem.get('name') == tableName:
            parent = Element(tableName)
            if iselement(parent):
                node = elem.find(tableNode)
                if node != None:
                    nodeElem = tableNode[:(len(tableNode) - 1)]
                    SubElement(node, nodeElem, data)
                if node == None:
                    node = SubElement(parent, tableNode)
                    nodeElem = tableNode[:(len(tableNode) - 1)]
                    SubElement(node, nodeElem, data)
        tree.write(xml_doc, xml_declaration=True, encoding='utf-8')
Esempio n. 17
0
def writeTableColumn(data,profile,category,tableName,tableNode):
    #Get new data to write to the config as a table column
    #node=None
    tree, root=parseRootElement()
    filters=(".//*[@name='%s']/%s")%(profile,category)
    for elem in root.findall(filters):
        if elem.get('name')==tableName:
            parent=Element(tableName)
            if iselement(parent):
                node=elem.find(tableNode)
                if node!=None:
                    nodeElem=tableNode[:(len(tableNode)-1)]
                    SubElement(node,nodeElem, data)
                if node==None:
                    node=SubElement(parent,tableNode)
                    nodeElem=tableNode[:(len(tableNode)-1)]
                    SubElement(node,nodeElem, data)
        tree.write(xml_doc,xml_declaration=True, encoding='utf-8')
Esempio n. 18
0
    def __init__(self, xml=None):
        self._user = None
        if xml is None:
            self._id = None
            self._name = None
            self._type = None
            self._properties = {}
            self._permissions = []
        elif isinstance(xml, str) or isinstance(xml, bytes) or iselement(xml):
            if isinstance(xml, str) or isinstance(xml, bytes):
                root = ElementTree.fromstring(xml)
            else:
                root = xml
            if root.tag != "report":
                raise PerfRepoException("Invalid xml.")

            self._id = root.get("id")
            self._name = root.get("name")
            self._type = root.get("type")
            self._properties = {}
            for entry in root.find("properties"):
                if entry.tag != "entry":
                    continue
                key_tag = entry.find("key")
                value_tag = entry.find("value")
                tmp_dict = dot_to_dict(value_tag.get("name"),
                                       value_tag.get("value"))
                recursive_dict_update(self._properties, tmp_dict)

            self._permissions = []
            for entry in root.find("permissions"):
                if entry.tag != "permission":
                    continue
                self._permissions.append(PerfRepoReportPermission(entry))
                self._permissions[-1].set_report_id(self._id)
                self._permissions[-1].validate()
        else:
            raise PerfRepoException("Parameter xml must be"\
                                    " a string, an Element or None")
Esempio n. 19
0
    def parse_error(self, msg=None):
        error_msg = 'Unknown error'

        try:
            # Azure does give some meaningful errors, but is inconsistent
            # Some APIs respond with an XML error. Others just dump HTML
            body = self.parse_body()

            if iselement(body):
                error_msg = parse_error(body)

        except MalformedResponseError:
            pass

        if msg:
            error_msg = '%s - %s' % (msg, error_msg)

        if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]:
            raise InvalidCredsError(error_msg)

        raise LibcloudError('%s Status code: %d.' % (error_msg, self.status),
                            driver=self)
Esempio n. 20
0
    def __init__(self, xml=None):
        self._report_id = None
        self._access_type = None
        self._access_level = None
        self._user_id = None
        self._group_id = None
        if xml is None:
            pass
        elif isinstance(xml, str) or isinstance(xml, bytes) or iselement(xml):
            if isinstance(xml, str) or isinstance(xml, bytes):
                root = ElementTree.fromstring(xml)
            else:
                root = xml
            if root.tag != "report-permission" and root.tag != "permission":
                raise PerfRepoException("Invalid xml.")

            self._id = root.find("id").text

            try:
                self._report_id = root.find("report-id").text
            except:
                self._report_id = None

            self._access_type = root.find("access-type").text
            self._access_level = root.find("access-level").text

            try:
                self._user_id = root.find("user-id").text
            except:
                self._user_id = None
            try:
                self._group_id = root.find("group-id").text
            except:
                self._group_id = None

        else:
            raise PerfRepoException("Parameter xml must be"\
                                    " a string, an Element or None")
    def __init__(self, xml=None):
        self._ids = None
        self._testUid = None
        self._testname = None
        self._tags = []
        self._parameters = []
        self._after = None
        self._before = None
        self._howmany = None

        if isinstance(xml, str) or isinstance(xml, bytes) or iselement(xml):
            if isinstance(xml, str) or isinstance(xml, bytes):
                root = ElementTree.fromstring(xml)
            else:
                root = xml
            if root.tag != "test-execution-search":
                raise PerfRepoException("Invalid xml.")

            elem_ids = root.find("ids")
            if elem_ids is not None:
                ids = []
                for id in elem_ids:
                    ids.append(id.text)
                self.set_ids(ids)

            self._testname = root.get("test-name")
            self._testUid = root.get("test-uid")
            elem_tags = root.find("tags")
            if elem_tags is not None and elem_tags.text is not None:
                tags = root.find("tags").text
                self._tags = tags.split()

            params = root.find("parameters")
            for param in params.findall("parameter"):
                pname = param.find("name")
                pvalue = param.find("value")
                self.add_parameter(pname.text, pvalue.text)
Esempio n. 22
0
def xml_to_csv(xml_tree) -> str:
    """
    From given xml tree, create new csv file
    :param xml_tree: XML tree
    :return: csv file name
    """
    csv_header = [
        'FinInstrmGnlAttrbts.Id', 'FinInstrmGnlAttrbts.FullNm',
        'FinInstrmGnlAttrbts.ClssfctnTp', 'FinInstrmGnlAttrbts.CmmdtyDerivInd',
        'FinInstrmGnlAttrbts.NtnlCcy', 'Issr'
    ]
    try:
        with open('file.csv', 'w') as csv_dict_writer:
            writer = csv.DictWriter(csv_dict_writer, fieldnames=csv_header)
            writer.writeheader()

            for record in xml_tree.iter('TermntdRcrd'):
                main_block = record.find('FinInstrmGnlAttrbts')
                data_dict = dict()
                if iselement(main_block):
                    data_dict['FinInstrmGnlAttrbts.Id'] = main_block.find("Id").text \
                        if iselement(main_block.find("Id")) else None
                    data_dict['FinInstrmGnlAttrbts.FullNm'] = main_block.find("FullNm").text \
                        if iselement(main_block.find("FullNm")) else None
                    data_dict['FinInstrmGnlAttrbts.ClssfctnTp'] = main_block.find("ClssfctnTp").text \
                        if iselement(main_block.find("ClssfctnTp")) else None
                    data_dict['FinInstrmGnlAttrbts.CmmdtyDerivInd'] = main_block.find("CmmdtyDerivInd").text \
                        if iselement(main_block.find("CmmdtyDerivInd")) else None
                    data_dict['FinInstrmGnlAttrbts.NtnlCcy'] = main_block.find("NtnlCcy").text \
                        if iselement(main_block.find("NtnlCcy")) else None
                data_dict['Issr'] = record.find('Issr').text if iselement(
                    record.find('Issr')) else None
                writer.writerow(data_dict)
        logger.info('Successfully csv fle is created {file.csv}')
        return 'file.csv'
    except IOError:
        logger.error("Not able to create a csv file or IO bound error",
                     exc_info=True)
        raise IOError