def __init__(self, xml=None): if type(xml) is NoneType: self._metricComparator = None self._metricName = None self._result = None self._parameters = [] elif type(xml) is StringType or iselement(xml): if type(xml) is StringType: root = ElementTree.fromstring(xml) elif iselement(xml): root = xml if root.tag != "value": raise PerfRepoException("Invalid xml.") self._metricComparator = root.get("metricComparator") self._metricName = root.get("metricName") self._result = float(root.get("result")) self._parameters = [] for param in root.find("parameters"): if param.tag != "parameter": continue self._parameters.append((param.get("name"), param.get("value"))) else: raise PerfRepoException("Parameter xml must be"\ " a string, an Element or None")
def __init__(self, xml=None): self._user = None if type(xml) is NoneType: self._id = None self._name = None self._type = None self._properties = {} elif type(xml) is StringType or iselement(xml): if type(xml) is StringType: root = ElementTree.fromstring(xml) elif iselement(xml): root = xml if root.tag != "report": raise PerfRepoException("Invalid xml.") self._id = root.get("id") self._name = root.get("name") self._type = root.get("type") self._properties = {} for entry in root.find("properties"): if entry.tag != "entry": continue key_tag = entry.find("key") value_tag = entry.find("value") tmp_dict = dot_to_dict(value_tag.get("name"), value_tag.get("value")) recursive_dict_update(self._properties, tmp_dict) else: raise PerfRepoException("Parameter xml must be"\ " a string, an Element or None")
def __init__(self, xml=None): if type(xml) is NoneType: self._id = None self._name = None self._uid = None self._description = None self._groupid = None self._metrics = [] elif type(xml) is StringType or iselement(xml): if type(xml) is StringType: root = ElementTree.fromstring(xml) elif iselement(xml): root = xml if root.tag != "test": raise PerfRepoException("Invalid xml.") self._id = root.get("id") self._name = root.get("name") self._uid = root.get("uid") self._groupid = root.get("groupId") self._description = root.find("description").text self._metrics = [] for metric in root.find("metrics"): if metric.tag != "metric": continue self._metrics.append(PerfRepoMetric(metric)) else: raise PerfRepoException("Parameter xml must be"\ " a string, an Element or None")
def parserMascotXMLAndWriteToFile(inputFileName, outputFileName): try: with open(outputFileName, 'w') as fw: fw.write('\t'.join(COLNAMES) + '\n') tree = ET.ElementTree(file=inputFileName) for elem in tree.iterfind(tag_queries_query_q_peptide): query_num = elem.attrib['query'] query_rank = elem.attrib['rank'] # queries -> query -> q_peptide tag_pep_seq = TAG_PREFIX + 'pep_seq' tag_pep_score = TAG_PREFIX + 'pep_score' tag_pep_var_mod = TAG_PREFIX + 'pep_var_mod' # Phospho tag_pep_var_mod_pos = TAG_PREFIX + 'pep_var_mod_pos' tag_pep_var_mod_conf = TAG_PREFIX + 'pep_var_mod_conf' e_tag_pep_seq = elem.find(tag_pep_seq) e_tag_pep_score = elem.find(tag_pep_score) e_tag_pep_var_mod = elem.find(tag_pep_var_mod) e_tag_pep_var_mod_pos = elem.find(tag_pep_var_mod_pos) e_tag_pep_var_mod_conf = elem.find(tag_pep_var_mod_conf) pep_seq = e_tag_pep_seq.text pep_score = e_tag_pep_score.text if iselement(e_tag_pep_var_mod ) and e_tag_pep_var_mod.text is not None: pep_var_mod = e_tag_pep_var_mod.text else: pep_var_mod = '' if iselement(e_tag_pep_var_mod_pos ) and e_tag_pep_var_mod_pos.text is not None: pep_var_mod_pos = e_tag_pep_var_mod_pos.text else: pep_var_mod_pos = '' if iselement(e_tag_pep_var_mod_conf ) and e_tag_pep_var_mod_conf.text is not None: pep_var_mod_conf = e_tag_pep_var_mod_conf.text else: pep_var_mod_conf = '' if pep_var_mod_pos != '': pep_seq = formatting_pep_seq(pep_seq, pep_var_mod_pos) lst = [ pep_seq, query_num, query_rank, pep_score, pep_var_mod, pep_var_mod_pos, pep_var_mod_conf ] lst_str = '\t'.join(lst) fw.write(lst_str + '\n') except RuntimeError: return False else: return True
def download_main_file(): """ The main function to handle all :return: """ url = "https://registers.esma.europa.eu/solr/esma_registers_firds_files/select" params = { "q": "*", "fq": "publication_date:[2021-01-17T00:00:00Z TO 2021-01-19T23:59:59Z]", "wt": "xml", "indent": "true", "start": "0", "rows": "100" } try: response = requests.get(url=url, params=params) logger.info(f"Successfully request is connected {url}") except ConnectionError: logger.error("Not able to connect to internet", exc_info=True) return False if response.status_code == 200: tree = ElementTree.fromstring(response.content.decode()) all_docs = tree.find('result').findall("doc") result_doc = None for doc in all_docs: if doc.find("str[@name='file_type']").text == 'DLTINS': result_doc = doc break if not iselement(result_doc): logger.info("Did not get any file type as `DLTINS`") return False download_doc = result_doc.find("str[@name='download_link']") if iselement(download_doc): download_link = download_doc.text xml_tree_root = get_xml_tree_from_zip(download_link) if not iselement(xml_tree_root): logger.info("Zip is not converted to xml tree") return False csv_file = xml_to_csv(xml_tree_root) if not csv_file: return False # TODO: uncomment below line to upload s3 bucket # status = upload_to_s3(csv_file) # logger.info("All task is done") # return status else: logger.warning("Did not found any download link to download zip") return False else: logger.error( f"Request is not proper. got status code ={response.status_code}, {response.content}", exc_info=True) return False
def __init__(self, xml=None): if type(xml) is NoneType: self._id = None self._name = None self._started = datetime.datetime.utcnow().isoformat() self._testId = None self._testUid = None self._comment = "" self._values = [] self._tags = [] self._parameters = [] elif type(xml) is StringType or iselement(xml): if type(xml) is StringType: root = ElementTree.fromstring(xml) elif iselement(xml): root = xml if root.tag != "testExecution": raise PerfRepoException("Invalid xml.") self._id = root.get("id") self._name = root.get("name") self._started = root.get("started") self._testId = root.get("testId") self._testUid = root.get("testUid") self._comment = root.find("comment").text self._values = [] for value in root.find("values"): if value.tag != "value": continue self._values.append(PerfRepoValue(value)) self._tags = [] for tag in root.find("tags"): if tag.tag != "tag": continue self._tags.append(tag.get("name")) self._parameters = [] for param in root.find("parameters"): if param.tag != "parameter": continue self._parameters.append((param.get("name"), param.get("value"))) else: raise PerfRepoException("Parameter xml must be"\ " a string, an Element or None")
def __init__(self, xml=None): self._report_id = None self._access_type = None self._access_level = None self._user_id = None self._group_id = None if type(xml) is NoneType: pass elif type(xml) is StringType or iselement(xml): if type(xml) is StringType: root = ElementTree.fromstring(xml) else: root = xml if root.tag != "report-permission": raise PerfRepoException("Invalid xml.") self._report_id = root.get("report-id").text self._access_type = root.get("access-type").text self._access_level = root.get("access-level").text try: self._user_id = root.get("user-id").text except: self._user_id = None try: self._group_id = root.get("group-id").text except: self._group_id = None else: raise PerfRepoException("Parameter xml must be"\ " a string, an Element or None")
def __init__(self, xml=None): self._ids = None self._testUid = None self._testname = None self._tags = [] self._parameters = [] if type(xml) is StringType or iselement(xml): if type(xml) is StringType: root = ElementTree.fromstring(xml) else: root = xml if root.tag != "test-execution-search": raise PerfRepoException("Invalid xml.") elem_ids = root.find("ids") if elem_ids is not None: ids = [] for id in elem_ids: ids.append(id.text) self.set_ids(ids) self._testname = root.get("test-name") self._testUid = root.get("test-uid") elem_tags = root.find("tags") if elem_tags is not None and elem_tags.text is not None: tags = root.find("tags").text self._tags = tags.split() params = root.find("parameters") for param in params.findall("parameter"): pname = param.find("name") pvalue = param.find("value") self.add_parameter(pname.text, pvalue.text)
def xmleval(source_tree, target_tree): """ Searches for the target_tree in the XML source tree Returns the number of occurance and location of each occurrence. target_string: the string of query element tree goal_list: a list of target_tree found in source tree. output_list is the list of output: first element in the list denotes the number of occurance other elements denote the location(id) of each occurance source_tree2 : created by inserting source tree in an empty element tree. the reason to do so is to include the top layer of source tree in the searching. Using element.set() and element.get() to set and get attribute key on the element to value, which reduce algorithm complexity compared to previous nested searching for loop """ Ocurrence_counter = 0 output_list = [] output_list.append(Ocurrence_counter) Start_String = ".//" # Selects all subelements on all level beneath the current element target_string = get_Target_String(target_tree,Start_String) tag_list = source_tree.iter() for (location,tag) in enumerate (tag_list,1): tag.set('#', str(location)) ''' # --------------previous indexing methods, retired------------------ index_tag = 1 index_list = [] for entry in tag_list: index_list.append([index_tag, entry]) index_tag += 1 # ------------------------------------------------------------------ ''' source_tree2 = Element("") source_tree2.insert(0,source_tree) assert(iselement(source_tree2) == True) goal_list = (source_tree2.findall(target_string)) assert(type(goal_list) is list) for entry in goal_list: output_list.append(int(entry.get('#'))) Ocurrence_counter +=1 ''' # --------------previous indexing methods, retired------------------ for x in goal_list: for y in index_list: if x == y[1]: assert(type(y[0] is int)) output_list.append(y[0]) Ocurrence_counter +=1 # ------------------------------------------------------------------ ''' output_list[0] = Ocurrence_counter assert(type(output_list) is list) return output_list
def coerce_response(response): if isinstance(response, basestring): response = HTTPResponse(HTTPPreamble(headers={'Content-Type': 'text/html'}), body=response) elif isinstance(response, dict): response = HTTPResponse(HTTPPreamble(headers={'Content-Type': 'application/json'}), body=json.dumps(response)) elif iselement(response): xml = StringIO() ElementTree(response).write(xml) response = HTTPResponse(HTTPPreamble(headers={'Content-Type': 'application/xml'}), body=xml.getvalue()) return response
def parse_cards(self, chassis_element): if not iselement(chassis_element): raise Exception("%s is not an element." % chassis_element) card_list = list(chassis_element) card_info = {} for card in card_list: cardName = card.attrib['Value'] card_info[cardName] = {"aid": card.attrib['aid'], "Pon": card.attrib['PON']} return card_info
def parse_chassis(self, ne_element): if not iselement(ne_element): raise Exception("%s is not an element." % ne_element) chassis_list = list(ne_element) chassisInfo = {} for chassis in chassis_list: chassisName = chassis.attrib['Value'] chs_element = list(chassis)[0] cardInfo = self.parse_cards(chs_element) chassisInfo[chassisName] = {'shelfId': chassis.attrib['aid'], 'chassisNo': chassis.attrib['CHASSERNO'], 'chsCategory': chs_element.tag, 'cardInfo': cardInfo} return chassisInfo
def writeTableColumn(data,tableName): #Get new data to write to the config as a table column try: tree, root=parseRootElement() for elem in root.findall('table'): if elem.get('name')==tableName: parent=Element(tableName) if iselement(parent): node=elem.find("columns") SubElement(node,"column", data) tree.write(xml_doc) except: pass
def raise_for_response(self, response, valid_response): if response.status != valid_response: error_msg = response.body try: body = parse_xml(error_msg) if iselement(body): error_msg = parse_error(body) except XMLParseError: pass values = (response.error, error_msg, response.status) message = 'Message: %s, Body: %s, Status code: %s' % (values) raise LibcloudError(message, driver=self)
def __init__(self, xml=None): if type(xml) is NoneType: self._id = None self._name = None self._description = None self._comparator = None elif type(xml) is StringType or iselement(xml): if type(xml) is StringType: root = ElementTree.fromstring(xml) elif iselement(xml): root = xml if root.tag != "metric": raise PerfRepoException("Invalid xml.") self._id = root.get("id") self._name = root.get("name") self._comparator = root.get("comparator") self._description = root.find("description").text else: raise PerfRepoException("Parameter xml must be"\ " a string, an Element or None")
def writeTableColumn(data, profile, category, tableName, tableNode): #Get new data to write to the config as a table column #node=None tree, root = parseRootElement() filters = (".//*[@name='%s']/%s") % (profile, category) for elem in root.findall(filters): if elem.get('name') == tableName: parent = Element(tableName) if iselement(parent): node = elem.find(tableNode) if node != None: nodeElem = tableNode[:(len(tableNode) - 1)] SubElement(node, nodeElem, data) if node == None: node = SubElement(parent, tableNode) nodeElem = tableNode[:(len(tableNode) - 1)] SubElement(node, nodeElem, data) tree.write(xml_doc, xml_declaration=True, encoding='utf-8')
def writeTableColumn(data,profile,category,tableName,tableNode): #Get new data to write to the config as a table column #node=None tree, root=parseRootElement() filters=(".//*[@name='%s']/%s")%(profile,category) for elem in root.findall(filters): if elem.get('name')==tableName: parent=Element(tableName) if iselement(parent): node=elem.find(tableNode) if node!=None: nodeElem=tableNode[:(len(tableNode)-1)] SubElement(node,nodeElem, data) if node==None: node=SubElement(parent,tableNode) nodeElem=tableNode[:(len(tableNode)-1)] SubElement(node,nodeElem, data) tree.write(xml_doc,xml_declaration=True, encoding='utf-8')
def __init__(self, xml=None): self._user = None if xml is None: self._id = None self._name = None self._type = None self._properties = {} self._permissions = [] elif isinstance(xml, str) or isinstance(xml, bytes) or iselement(xml): if isinstance(xml, str) or isinstance(xml, bytes): root = ElementTree.fromstring(xml) else: root = xml if root.tag != "report": raise PerfRepoException("Invalid xml.") self._id = root.get("id") self._name = root.get("name") self._type = root.get("type") self._properties = {} for entry in root.find("properties"): if entry.tag != "entry": continue key_tag = entry.find("key") value_tag = entry.find("value") tmp_dict = dot_to_dict(value_tag.get("name"), value_tag.get("value")) recursive_dict_update(self._properties, tmp_dict) self._permissions = [] for entry in root.find("permissions"): if entry.tag != "permission": continue self._permissions.append(PerfRepoReportPermission(entry)) self._permissions[-1].set_report_id(self._id) self._permissions[-1].validate() else: raise PerfRepoException("Parameter xml must be"\ " a string, an Element or None")
def parse_error(self, msg=None): error_msg = 'Unknown error' try: # Azure does give some meaningful errors, but is inconsistent # Some APIs respond with an XML error. Others just dump HTML body = self.parse_body() if iselement(body): error_msg = parse_error(body) except MalformedResponseError: pass if msg: error_msg = '%s - %s' % (msg, error_msg) if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]: raise InvalidCredsError(error_msg) raise LibcloudError('%s Status code: %d.' % (error_msg, self.status), driver=self)
def __init__(self, xml=None): self._report_id = None self._access_type = None self._access_level = None self._user_id = None self._group_id = None if xml is None: pass elif isinstance(xml, str) or isinstance(xml, bytes) or iselement(xml): if isinstance(xml, str) or isinstance(xml, bytes): root = ElementTree.fromstring(xml) else: root = xml if root.tag != "report-permission" and root.tag != "permission": raise PerfRepoException("Invalid xml.") self._id = root.find("id").text try: self._report_id = root.find("report-id").text except: self._report_id = None self._access_type = root.find("access-type").text self._access_level = root.find("access-level").text try: self._user_id = root.find("user-id").text except: self._user_id = None try: self._group_id = root.find("group-id").text except: self._group_id = None else: raise PerfRepoException("Parameter xml must be"\ " a string, an Element or None")
def __init__(self, xml=None): self._ids = None self._testUid = None self._testname = None self._tags = [] self._parameters = [] self._after = None self._before = None self._howmany = None if isinstance(xml, str) or isinstance(xml, bytes) or iselement(xml): if isinstance(xml, str) or isinstance(xml, bytes): root = ElementTree.fromstring(xml) else: root = xml if root.tag != "test-execution-search": raise PerfRepoException("Invalid xml.") elem_ids = root.find("ids") if elem_ids is not None: ids = [] for id in elem_ids: ids.append(id.text) self.set_ids(ids) self._testname = root.get("test-name") self._testUid = root.get("test-uid") elem_tags = root.find("tags") if elem_tags is not None and elem_tags.text is not None: tags = root.find("tags").text self._tags = tags.split() params = root.find("parameters") for param in params.findall("parameter"): pname = param.find("name") pvalue = param.find("value") self.add_parameter(pname.text, pvalue.text)
def xml_to_csv(xml_tree) -> str: """ From given xml tree, create new csv file :param xml_tree: XML tree :return: csv file name """ csv_header = [ 'FinInstrmGnlAttrbts.Id', 'FinInstrmGnlAttrbts.FullNm', 'FinInstrmGnlAttrbts.ClssfctnTp', 'FinInstrmGnlAttrbts.CmmdtyDerivInd', 'FinInstrmGnlAttrbts.NtnlCcy', 'Issr' ] try: with open('file.csv', 'w') as csv_dict_writer: writer = csv.DictWriter(csv_dict_writer, fieldnames=csv_header) writer.writeheader() for record in xml_tree.iter('TermntdRcrd'): main_block = record.find('FinInstrmGnlAttrbts') data_dict = dict() if iselement(main_block): data_dict['FinInstrmGnlAttrbts.Id'] = main_block.find("Id").text \ if iselement(main_block.find("Id")) else None data_dict['FinInstrmGnlAttrbts.FullNm'] = main_block.find("FullNm").text \ if iselement(main_block.find("FullNm")) else None data_dict['FinInstrmGnlAttrbts.ClssfctnTp'] = main_block.find("ClssfctnTp").text \ if iselement(main_block.find("ClssfctnTp")) else None data_dict['FinInstrmGnlAttrbts.CmmdtyDerivInd'] = main_block.find("CmmdtyDerivInd").text \ if iselement(main_block.find("CmmdtyDerivInd")) else None data_dict['FinInstrmGnlAttrbts.NtnlCcy'] = main_block.find("NtnlCcy").text \ if iselement(main_block.find("NtnlCcy")) else None data_dict['Issr'] = record.find('Issr').text if iselement( record.find('Issr')) else None writer.writerow(data_dict) logger.info('Successfully csv fle is created {file.csv}') return 'file.csv' except IOError: logger.error("Not able to create a csv file or IO bound error", exc_info=True) raise IOError