def __init__(self, xml=None):
if type(xml) is NoneType:
self._metricComparator = None
self._metricName = None
self._result = None
self._parameters = []
elif type(xml) is StringType or iselement(xml):
if type(xml) is StringType:
root = ElementTree.fromstring(xml)
elif iselement(xml):
root = xml
if root.tag != "value":
raise PerfRepoException("Invalid xml.")
self._metricComparator = root.get("metricComparator")
self._metricName = root.get("metricName")
self._result = float(root.get("result"))
self._parameters = []
for param in root.find("parameters"):
if param.tag != "parameter":
continue
self._parameters.append((param.get("name"), param.get("value")))
else:
raise PerfRepoException("Parameter xml must be"\
" a string, an Element or None")
def __init__(self, xml=None):
self._user = None
if type(xml) is NoneType:
self._id = None
self._name = None
self._type = None
self._properties = {}
elif type(xml) is StringType or iselement(xml):
if type(xml) is StringType:
root = ElementTree.fromstring(xml)
elif iselement(xml):
root = xml
if root.tag != "report":
raise PerfRepoException("Invalid xml.")
self._id = root.get("id")
self._name = root.get("name")
self._type = root.get("type")
self._properties = {}
for entry in root.find("properties"):
if entry.tag != "entry":
continue
key_tag = entry.find("key")
value_tag = entry.find("value")
tmp_dict = dot_to_dict(value_tag.get("name"),
value_tag.get("value"))
recursive_dict_update(self._properties, tmp_dict)
else:
raise PerfRepoException("Parameter xml must be"\
" a string, an Element or None")
def __init__(self, xml=None):
if type(xml) is NoneType:
self._id = None
self._name = None
self._uid = None
self._description = None
self._groupid = None
self._metrics = []
elif type(xml) is StringType or iselement(xml):
if type(xml) is StringType:
root = ElementTree.fromstring(xml)
elif iselement(xml):
root = xml
if root.tag != "test":
raise PerfRepoException("Invalid xml.")
self._id = root.get("id")
self._name = root.get("name")
self._uid = root.get("uid")
self._groupid = root.get("groupId")
self._description = root.find("description").text
self._metrics = []
for metric in root.find("metrics"):
if metric.tag != "metric":
continue
self._metrics.append(PerfRepoMetric(metric))
else:
raise PerfRepoException("Parameter xml must be"\
" a string, an Element or None")
def parserMascotXMLAndWriteToFile(inputFileName, outputFileName):
try:
with open(outputFileName, 'w') as fw:
fw.write('\t'.join(COLNAMES) + '\n')
tree = ET.ElementTree(file=inputFileName)
for elem in tree.iterfind(tag_queries_query_q_peptide):
query_num = elem.attrib['query']
query_rank = elem.attrib['rank']
# queries -> query -> q_peptide
tag_pep_seq = TAG_PREFIX + 'pep_seq'
tag_pep_score = TAG_PREFIX + 'pep_score'
tag_pep_var_mod = TAG_PREFIX + 'pep_var_mod' # Phospho
tag_pep_var_mod_pos = TAG_PREFIX + 'pep_var_mod_pos'
tag_pep_var_mod_conf = TAG_PREFIX + 'pep_var_mod_conf'
e_tag_pep_seq = elem.find(tag_pep_seq)
e_tag_pep_score = elem.find(tag_pep_score)
e_tag_pep_var_mod = elem.find(tag_pep_var_mod)
e_tag_pep_var_mod_pos = elem.find(tag_pep_var_mod_pos)
e_tag_pep_var_mod_conf = elem.find(tag_pep_var_mod_conf)
pep_seq = e_tag_pep_seq.text
pep_score = e_tag_pep_score.text
if iselement(e_tag_pep_var_mod
) and e_tag_pep_var_mod.text is not None:
pep_var_mod = e_tag_pep_var_mod.text
else:
pep_var_mod = ''
if iselement(e_tag_pep_var_mod_pos
) and e_tag_pep_var_mod_pos.text is not None:
pep_var_mod_pos = e_tag_pep_var_mod_pos.text
else:
pep_var_mod_pos = ''
if iselement(e_tag_pep_var_mod_conf
) and e_tag_pep_var_mod_conf.text is not None:
pep_var_mod_conf = e_tag_pep_var_mod_conf.text
else:
pep_var_mod_conf = ''
if pep_var_mod_pos != '':
pep_seq = formatting_pep_seq(pep_seq, pep_var_mod_pos)
lst = [
pep_seq, query_num, query_rank, pep_score, pep_var_mod,
pep_var_mod_pos, pep_var_mod_conf
]
lst_str = '\t'.join(lst)
fw.write(lst_str + '\n')
except RuntimeError:
return False
else:
return True
def download_main_file():
"""
The main function to handle all
:return:
"""
url = "https://registers.esma.europa.eu/solr/esma_registers_firds_files/select"
params = {
"q": "*",
"fq":
"publication_date:[2021-01-17T00:00:00Z TO 2021-01-19T23:59:59Z]",
"wt": "xml",
"indent": "true",
"start": "0",
"rows": "100"
}
try:
response = requests.get(url=url, params=params)
logger.info(f"Successfully request is connected {url}")
except ConnectionError:
logger.error("Not able to connect to internet", exc_info=True)
return False
if response.status_code == 200:
tree = ElementTree.fromstring(response.content.decode())
all_docs = tree.find('result').findall("doc")
result_doc = None
for doc in all_docs:
if doc.find("str[@name='file_type']").text == 'DLTINS':
result_doc = doc
break
if not iselement(result_doc):
logger.info("Did not get any file type as `DLTINS`")
return False
download_doc = result_doc.find("str[@name='download_link']")
if iselement(download_doc):
download_link = download_doc.text
xml_tree_root = get_xml_tree_from_zip(download_link)
if not iselement(xml_tree_root):
logger.info("Zip is not converted to xml tree")
return False
csv_file = xml_to_csv(xml_tree_root)
if not csv_file:
return False
# TODO: uncomment below line to upload s3 bucket
# status = upload_to_s3(csv_file)
# logger.info("All task is done")
# return status
else:
logger.warning("Did not found any download link to download zip")
return False
else:
logger.error(
f"Request is not proper. got status code ={response.status_code}, {response.content}",
exc_info=True)
return False
def __init__(self, xml=None):
if type(xml) is NoneType:
self._id = None
self._name = None
self._started = datetime.datetime.utcnow().isoformat()
self._testId = None
self._testUid = None
self._comment = ""
self._values = []
self._tags = []
self._parameters = []
elif type(xml) is StringType or iselement(xml):
if type(xml) is StringType:
root = ElementTree.fromstring(xml)
elif iselement(xml):
root = xml
if root.tag != "testExecution":
raise PerfRepoException("Invalid xml.")
self._id = root.get("id")
self._name = root.get("name")
self._started = root.get("started")
self._testId = root.get("testId")
self._testUid = root.get("testUid")
self._comment = root.find("comment").text
self._values = []
for value in root.find("values"):
if value.tag != "value":
continue
self._values.append(PerfRepoValue(value))
self._tags = []
for tag in root.find("tags"):
if tag.tag != "tag":
continue
self._tags.append(tag.get("name"))
self._parameters = []
for param in root.find("parameters"):
if param.tag != "parameter":
continue
self._parameters.append((param.get("name"), param.get("value")))
else:
raise PerfRepoException("Parameter xml must be"\
" a string, an Element or None")
def __init__(self, xml=None):
self._report_id = None
self._access_type = None
self._access_level = None
self._user_id = None
self._group_id = None
if type(xml) is NoneType:
pass
elif type(xml) is StringType or iselement(xml):
if type(xml) is StringType:
root = ElementTree.fromstring(xml)
else:
root = xml
if root.tag != "report-permission":
raise PerfRepoException("Invalid xml.")
self._report_id = root.get("report-id").text
self._access_type = root.get("access-type").text
self._access_level = root.get("access-level").text
try:
self._user_id = root.get("user-id").text
except:
self._user_id = None
try:
self._group_id = root.get("group-id").text
except:
self._group_id = None
else:
raise PerfRepoException("Parameter xml must be"\
" a string, an Element or None")
def __init__(self, xml=None):
self._ids = None
self._testUid = None
self._testname = None
self._tags = []
self._parameters = []
if type(xml) is StringType or iselement(xml):
if type(xml) is StringType:
root = ElementTree.fromstring(xml)
else:
root = xml
if root.tag != "test-execution-search":
raise PerfRepoException("Invalid xml.")
elem_ids = root.find("ids")
if elem_ids is not None:
ids = []
for id in elem_ids:
ids.append(id.text)
self.set_ids(ids)
self._testname = root.get("test-name")
self._testUid = root.get("test-uid")
elem_tags = root.find("tags")
if elem_tags is not None and elem_tags.text is not None:
tags = root.find("tags").text
self._tags = tags.split()
params = root.find("parameters")
for param in params.findall("parameter"):
pname = param.find("name")
pvalue = param.find("value")
self.add_parameter(pname.text, pvalue.text)
def xmleval(source_tree, target_tree):
"""
Searches for the target_tree in the XML source tree
Returns the number of occurance and location of each occurrence.
target_string: the string of query element tree
goal_list: a list of target_tree found in source tree.
output_list is the list of output:
first element in the list denotes the number of occurance
other elements denote the location(id) of each occurance
source_tree2 : created by inserting source tree in an empty element tree.
the reason to do so is to include the top layer of source tree in the searching.
Using element.set() and element.get() to set and get attribute key on the element to value,
which reduce algorithm complexity compared to previous nested searching for loop
"""
Ocurrence_counter = 0
output_list = []
output_list.append(Ocurrence_counter)
Start_String = ".//" # Selects all subelements on all level beneath the current element
target_string = get_Target_String(target_tree,Start_String)
tag_list = source_tree.iter()
for (location,tag) in enumerate (tag_list,1):
tag.set('#', str(location))
'''
# --------------previous indexing methods, retired------------------
index_tag = 1
index_list = []
for entry in tag_list:
index_list.append([index_tag, entry])
index_tag += 1
# ------------------------------------------------------------------
'''
source_tree2 = Element("")
source_tree2.insert(0,source_tree)
assert(iselement(source_tree2) == True)
goal_list = (source_tree2.findall(target_string))
assert(type(goal_list) is list)
for entry in goal_list:
output_list.append(int(entry.get('#')))
Ocurrence_counter +=1
'''
# --------------previous indexing methods, retired------------------
for x in goal_list:
for y in index_list:
if x == y[1]:
assert(type(y[0] is int))
output_list.append(y[0])
Ocurrence_counter +=1
# ------------------------------------------------------------------
'''
output_list[0] = Ocurrence_counter
assert(type(output_list) is list)
return output_list
def coerce_response(response):
if isinstance(response, basestring):
response = HTTPResponse(HTTPPreamble(headers={'Content-Type': 'text/html'}), body=response)
elif isinstance(response, dict):
response = HTTPResponse(HTTPPreamble(headers={'Content-Type': 'application/json'}), body=json.dumps(response))
elif iselement(response):
xml = StringIO()
ElementTree(response).write(xml)
response = HTTPResponse(HTTPPreamble(headers={'Content-Type': 'application/xml'}), body=xml.getvalue())
return response
def parse_cards(self, chassis_element):
if not iselement(chassis_element):
raise Exception("%s is not an element." % chassis_element)
card_list = list(chassis_element)
card_info = {}
for card in card_list:
cardName = card.attrib['Value']
card_info[cardName] = {"aid": card.attrib['aid'], "Pon": card.attrib['PON']}
return card_info
def parse_chassis(self, ne_element):
if not iselement(ne_element):
raise Exception("%s is not an element." % ne_element)
chassis_list = list(ne_element)
chassisInfo = {}
for chassis in chassis_list:
chassisName = chassis.attrib['Value']
chs_element = list(chassis)[0]
cardInfo = self.parse_cards(chs_element)
chassisInfo[chassisName] = {'shelfId': chassis.attrib['aid'], 'chassisNo': chassis.attrib['CHASSERNO'],
'chsCategory': chs_element.tag, 'cardInfo': cardInfo}
return chassisInfo
def writeTableColumn(data,tableName):
#Get new data to write to the config as a table column
try:
tree, root=parseRootElement()
for elem in root.findall('table'):
if elem.get('name')==tableName:
parent=Element(tableName)
if iselement(parent):
node=elem.find("columns")
SubElement(node,"column", data)
tree.write(xml_doc)
except:
pass
def raise_for_response(self, response, valid_response):
if response.status != valid_response:
error_msg = response.body
try:
body = parse_xml(error_msg)
if iselement(body):
error_msg = parse_error(body)
except XMLParseError:
pass
values = (response.error, error_msg, response.status)
message = 'Message: %s, Body: %s, Status code: %s' % (values)
raise LibcloudError(message, driver=self)
def __init__(self, xml=None):
if type(xml) is NoneType:
self._id = None
self._name = None
self._description = None
self._comparator = None
elif type(xml) is StringType or iselement(xml):
if type(xml) is StringType:
root = ElementTree.fromstring(xml)
elif iselement(xml):
root = xml
if root.tag != "metric":
raise PerfRepoException("Invalid xml.")
self._id = root.get("id")
self._name = root.get("name")
self._comparator = root.get("comparator")
self._description = root.find("description").text
else:
raise PerfRepoException("Parameter xml must be"\
" a string, an Element or None")
def writeTableColumn(data, profile, category, tableName, tableNode):
#Get new data to write to the config as a table column
#node=None
tree, root = parseRootElement()
filters = (".//*[@name='%s']/%s") % (profile, category)
for elem in root.findall(filters):
if elem.get('name') == tableName:
parent = Element(tableName)
if iselement(parent):
node = elem.find(tableNode)
if node != None:
nodeElem = tableNode[:(len(tableNode) - 1)]
SubElement(node, nodeElem, data)
if node == None:
node = SubElement(parent, tableNode)
nodeElem = tableNode[:(len(tableNode) - 1)]
SubElement(node, nodeElem, data)
tree.write(xml_doc, xml_declaration=True, encoding='utf-8')
def writeTableColumn(data,profile,category,tableName,tableNode):
#Get new data to write to the config as a table column
#node=None
tree, root=parseRootElement()
filters=(".//*[@name='%s']/%s")%(profile,category)
for elem in root.findall(filters):
if elem.get('name')==tableName:
parent=Element(tableName)
if iselement(parent):
node=elem.find(tableNode)
if node!=None:
nodeElem=tableNode[:(len(tableNode)-1)]
SubElement(node,nodeElem, data)
if node==None:
node=SubElement(parent,tableNode)
nodeElem=tableNode[:(len(tableNode)-1)]
SubElement(node,nodeElem, data)
tree.write(xml_doc,xml_declaration=True, encoding='utf-8')
def __init__(self, xml=None):
self._user = None
if xml is None:
self._id = None
self._name = None
self._type = None
self._properties = {}
self._permissions = []
elif isinstance(xml, str) or isinstance(xml, bytes) or iselement(xml):
if isinstance(xml, str) or isinstance(xml, bytes):
root = ElementTree.fromstring(xml)
else:
root = xml
if root.tag != "report":
raise PerfRepoException("Invalid xml.")
self._id = root.get("id")
self._name = root.get("name")
self._type = root.get("type")
self._properties = {}
for entry in root.find("properties"):
if entry.tag != "entry":
continue
key_tag = entry.find("key")
value_tag = entry.find("value")
tmp_dict = dot_to_dict(value_tag.get("name"),
value_tag.get("value"))
recursive_dict_update(self._properties, tmp_dict)
self._permissions = []
for entry in root.find("permissions"):
if entry.tag != "permission":
continue
self._permissions.append(PerfRepoReportPermission(entry))
self._permissions[-1].set_report_id(self._id)
self._permissions[-1].validate()
else:
raise PerfRepoException("Parameter xml must be"\
" a string, an Element or None")
def parse_error(self, msg=None):
error_msg = 'Unknown error'
try:
# Azure does give some meaningful errors, but is inconsistent
# Some APIs respond with an XML error. Others just dump HTML
body = self.parse_body()
if iselement(body):
error_msg = parse_error(body)
except MalformedResponseError:
pass
if msg:
error_msg = '%s - %s' % (msg, error_msg)
if self.status in [httplib.UNAUTHORIZED, httplib.FORBIDDEN]:
raise InvalidCredsError(error_msg)
raise LibcloudError('%s Status code: %d.' % (error_msg, self.status),
driver=self)
def __init__(self, xml=None):
self._report_id = None
self._access_type = None
self._access_level = None
self._user_id = None
self._group_id = None
if xml is None:
pass
elif isinstance(xml, str) or isinstance(xml, bytes) or iselement(xml):
if isinstance(xml, str) or isinstance(xml, bytes):
root = ElementTree.fromstring(xml)
else:
root = xml
if root.tag != "report-permission" and root.tag != "permission":
raise PerfRepoException("Invalid xml.")
self._id = root.find("id").text
try:
self._report_id = root.find("report-id").text
except:
self._report_id = None
self._access_type = root.find("access-type").text
self._access_level = root.find("access-level").text
try:
self._user_id = root.find("user-id").text
except:
self._user_id = None
try:
self._group_id = root.find("group-id").text
except:
self._group_id = None
else:
raise PerfRepoException("Parameter xml must be"\
" a string, an Element or None")
def __init__(self, xml=None):
self._ids = None
self._testUid = None
self._testname = None
self._tags = []
self._parameters = []
self._after = None
self._before = None
self._howmany = None
if isinstance(xml, str) or isinstance(xml, bytes) or iselement(xml):
if isinstance(xml, str) or isinstance(xml, bytes):
root = ElementTree.fromstring(xml)
else:
root = xml
if root.tag != "test-execution-search":
raise PerfRepoException("Invalid xml.")
elem_ids = root.find("ids")
if elem_ids is not None:
ids = []
for id in elem_ids:
ids.append(id.text)
self.set_ids(ids)
self._testname = root.get("test-name")
self._testUid = root.get("test-uid")
elem_tags = root.find("tags")
if elem_tags is not None and elem_tags.text is not None:
tags = root.find("tags").text
self._tags = tags.split()
params = root.find("parameters")
for param in params.findall("parameter"):
pname = param.find("name")
pvalue = param.find("value")
self.add_parameter(pname.text, pvalue.text)
def xml_to_csv(xml_tree) -> str:
"""
From given xml tree, create new csv file
:param xml_tree: XML tree
:return: csv file name
"""
csv_header = [
'FinInstrmGnlAttrbts.Id', 'FinInstrmGnlAttrbts.FullNm',
'FinInstrmGnlAttrbts.ClssfctnTp', 'FinInstrmGnlAttrbts.CmmdtyDerivInd',
'FinInstrmGnlAttrbts.NtnlCcy', 'Issr'
]
try:
with open('file.csv', 'w') as csv_dict_writer:
writer = csv.DictWriter(csv_dict_writer, fieldnames=csv_header)
writer.writeheader()
for record in xml_tree.iter('TermntdRcrd'):
main_block = record.find('FinInstrmGnlAttrbts')
data_dict = dict()
if iselement(main_block):
data_dict['FinInstrmGnlAttrbts.Id'] = main_block.find("Id").text \
if iselement(main_block.find("Id")) else None
data_dict['FinInstrmGnlAttrbts.FullNm'] = main_block.find("FullNm").text \
if iselement(main_block.find("FullNm")) else None
data_dict['FinInstrmGnlAttrbts.ClssfctnTp'] = main_block.find("ClssfctnTp").text \
if iselement(main_block.find("ClssfctnTp")) else None
data_dict['FinInstrmGnlAttrbts.CmmdtyDerivInd'] = main_block.find("CmmdtyDerivInd").text \
if iselement(main_block.find("CmmdtyDerivInd")) else None
data_dict['FinInstrmGnlAttrbts.NtnlCcy'] = main_block.find("NtnlCcy").text \
if iselement(main_block.find("NtnlCcy")) else None
data_dict['Issr'] = record.find('Issr').text if iselement(
record.find('Issr')) else None
writer.writerow(data_dict)
logger.info('Successfully csv fle is created {file.csv}')
return 'file.csv'
except IOError:
logger.error("Not able to create a csv file or IO bound error",
exc_info=True)
raise IOError
