def main():
device_params=None
if aargs.devicetype in ['j','c','n','h','junos','csr','nexus','huawei']:
device_params={'name':aargs.devicetype} if len(aargs.devicetype)>1 else None
device_params={'name':'junos'} if not device_params and aargs.devicetype=='j' else None
device_params={'name':'csr'} if not device_params and aargs.devicetype=='c' else None
device_params={'name':'nexus'} if not device_params and aargs.devicetype=='n' else None
device_params={'name':'huawei'} if not device_params and aargs.devicetype=='h' else None
### READ YAML NETCONF AUTH FILE ----------------------------------------------
with open('./netconf_auth_data.yaml', 'r') as stream: netconf_auth_data = yaml.load(stream)
if netconf_auth_data:
with manager.connect_ssh(host=netconf_auth_data.get('netconf_ipaddress'),
port=netconf_auth_data.get('netconf_ssh_port'),
username=netconf_auth_data.get('netconf_user'),
password=netconf_auth_data.get('netconf_password'),
device_params=device_params,
hostkey_verify=False ) as m:
if aargs.verbose:
print('CONNECTED:',m.connected,'\nCAPABILITIES:')
for c in m.server_capabilities: print(c)
### GET RUNNING CONFIG =================================================
if aargs.getrunningconfig:
running_config = m.get_config('running').data_xml
if aargs.verbose: print('\nRUNNING_CONFIG:\n',xml.dom.minidom.parseString(str(running_config)).toprettyxml())
now = datetime.datetime.now()
timestring='%04d%02d%02d_%02d%02d%02d'%(now.year,now.month,now.day,now.hour,now.minute,now.second)
file_name='running_config_'+timestring+'.xml'
with open(file_name, 'w', encoding='utf8') as outfile:
outfile.write(xml.dom.minidom.parseString(str(running_config)).toprettyxml())
print('Writing running-config to file:',file_name)
#exit(0)
if aargs.comparewithfile:
diff = xdmain.diff_files(aargs.comparewithfile, file_name, formatter=formatting.XMLFormatter())
print(diff)
class HTMLFormatterFileTests(FormatterFileTests):
# We use a few tags for the placeholder tests.
# <br/> is intentionally left out, to test an edge case
# with empty non-formatting tags in text.
formatter = formatting.XMLFormatter(
normalize=formatting.WS_BOTH,
pretty_print=True,
text_tags=('p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'),
formatting_tags=('b', 'u', 'i', 'strike', 'em', 'super', 'sup', 'sub',
'link', 'a', 'span'))
def set_in_motion_find_diffs(file1name, file2name):
diff = ''
diff = main.diff_files(file1name, file2name, formatter=formatting.XMLFormatter())
for i in diff.splitlines():
if re.search(r'\bdiff:\w+', i) or i.startswith('</node') or i.startswith('<node'):
if not cull_line(i, 'TRUE'):
categorize_it(i)
get_all_node_from_keyfile(file1name) #double check the missing and extra list
get_all_node_from_studentfile(file2name)
key_file(file1name)
student_file(file2name)
compare(print_list_for_key, print_list_for_student)
double_check_moved() # double check the moved node using key and student links
key_crosslink(file1name)
student_crosslink(file2name)
compare_crosslink(print_list_key_crosslink, print_list_student_crosslink)
return
def compare(self, old: str, new: str):
old, new = self.preprocess(old, new)
old, new = get_xml(old), get_xml(new)
diffs = xml.diff_texts(old,
new,
diff_options={
'F': 0.5,
'ratio_mode': 'accurate'
})
diff = xml.diff_texts(
old,
new,
formatter=formatting.XMLFormatter(normalize=formatting.WS_BOTH),
diff_options={
'F': 0.5,
'ratio_mode': 'accurate'
})
diff = normalize_html(diff)
self.set_style(diff)
return diff.prettify()
class HTMLFormatterFileTests(FormatterFileTests):
# We use a few tags for the placeholder tests.
# <br/> is intentionally left out, to test an edge case
# with empty non-formatting tags in text.
formatter = formatting.XMLFormatter(
normalize=formatting.WS_BOTH,
pretty_print=True,
text_tags=("p", "h1", "h2", "h3", "h4", "h5", "h6", "li"),
formatting_tags=(
"b",
"u",
"i",
"strike",
"em",
"super",
"sup",
"sub",
"link",
"a",
"span",
),
)
)
stdin, stdout, stderr = ssh.exec_command(
'pmrep objectexport -o workflow -f Sales_Forecasting -n wf_Pipeline_EDW_WAVE_Scratch_Pad_Snapshot_Load_incremental -m -s -b -r -u /home/amedishetti/Artifact_Comparator/workflows/WORKFLOW_dev.xml'
)
print(stdout)
scp = SCPClient(ssh.get_transport())
scp.get(
'/apps/infa/PowerCenter8.6.1/server/infa_shared/ArtifactComparison/WORKFLOW.xml'
)
scp.get(
'/apps/infa/PowerCenter8.6.1/server/infa_shared/ArtifactComparison/WORKFLOW_dev.xml'
)
ssh.close()
diff = main.diff_files('WORKFLOW.xml',
'WORKFLOW_dev.xml',
formatter=formatting.XMLFormatter())
print(diff, file=open("test_compare.xml", 'w'))
#con = cx_Oracle.connect('INFAREP_PROD_READ/[email protected]:1521/INFPRD')
#cur = con.cursor()
#query1 = ("SELECT * FROM (SELECT WF.WORKFLOW_NAME, COUNT(TSK.INSTANCE_NAME) AS TSK_COUNT FROM INFAREP_PROD.REP_WORKFLOWS WF INNER JOIN INFAREP_PROD.REP_TASK_INST TSK ON WF.WORKFLOW_ID=TSK.WORKFLOW_ID GROUP BY WF.WORKFLOW_NAME) S WHERE S.WORKFLOW_NAME='"+s[0:-4]+"'")
#cur.execute(query1)
#for result in cur:
#print(result,file=open(file_prod,"a"))
#query2 = ("SELECT * FROM (SELECT WF.WORKFLOW_NAME, COUNT(TSK.INSTANCE_NAME) AS TSK_COUNT FROM INFAREP_"+z+".REP_WORKFLOWS WF INNER JOIN INFAREP_"+z+".REP_TASK_INST TSK ON WF.WORKFLOW_ID=TSK.WORKFLOW_ID GROUP BY WF.WORKFLOW_NAME) S WHERE S.WORKFLOW_NAME='"+s[0:-4]+"'")
#con = cx_Oracle.connect('infarep_dev15/[email protected]/infdev15')
#cur=con.cursor()
#cur.execute(query2)
#file_1 = set()
#file_2 = set()
#for result in cur:
#print(result,file=open(file_env, "w"))
class XMLFormatterFileTests(FormatterFileTests):
# The XMLFormatter has no text or formatting tags, so
formatter = formatting.XMLFormatter(pretty_print=False,
normalize=formatting.WS_TEXT)
def _format_test(self, left, action, expected):
formatter = formatting.XMLFormatter(pretty_print=False)
result = formatter.format([action], etree.fromstring(left))
self.assertEqual(result, expected)
def test_api_diff_files_with_formatter(self):
formatter = formatting.XMLFormatter()
# diff_files can take filenames
result = main.diff_files(LEFT_FILE, RIGHT_FILE, formatter=formatter)
# This formatter will insert a diff namespace:
self.assertIn('xmlns:diff="http://namespaces.shoobx.com/diff"', result)
def main():
recording_flag = False
activity_start_count = 0
query = sys.argv[1] + '&format=xml'
query_response = requests.get(query)
with open('activity-list.xml', 'wb') as activity_query_file:
activity_query_file.write(query_response.content)
with open('activity-list-fix.xml', 'w') as new_activity_file:
with open('activity-list.xml', 'r') as activity_file:
for line in activity_file:
if "<iati-activity" in line:
activity_start_count = activity_start_count + 1
if "iati-extra" in line:
line = line.replace(
'<iati-activity',
'<iati-activity xmlns:iati-extra="http://datastore.iatistandard.org/ns"'
)
if activity_start_count > 1:
line = '\n'
recording_flag = True
if recording_flag == True:
if "</iati-activity" in line or "</iati-activities" in line:
line = '\n'
else:
new_activity_file.write(line.strip() + "\n")
new_activity_file.write('</iati-activity>')
# LOAD XML AND XSL SCRIPT
xml = ET.parse('activity-list-fix.xml')
xsl = ET.parse('activity_style.xsl')
xslt_formatting = ET.parse('htmlformatter.xslt')
transform = ET.XSLT(xsl)
# LOOP THROUGH ALL NODE COUNTS AND PASS PARAMETER TO XSLT
iati_count = len(xml.xpath('//iati-identifier'))
formatter = formatting.XMLFormatter(normalize=formatting.WS_BOTH,
pretty_print=True)
for i in range(iati_count):
newf = ""
newf2 = ""
n = ET.XSLT.strparam(str(i + 1))
result = transform(xml, item_num=n) # NAME OF XSL PARAMETER
# SAVE XML TO FILE
with open(path_activities + 'Output_{}.xml'.format(i + 1), 'wb') as f:
#f.write(header.encode('utf-8'))
f.write(result)
output_parser = ET.parse(path_activities +
'Output_{}.xml'.format(i + 1))
output_root = output_parser.getroot()
output_identifier = output_root.find('iati-identifier')
datastore_xml_url = 'http://datastore.iatistandard.org/api/1/access/activity.xml?iati-identifier=' + output_identifier.text
response = requests.get(datastore_xml_url)
with open(path_datastore + output_identifier.text + '.xml',
'wb') as file:
xml_ugly = prettifier.parseString(response.content)
xml_pretty = xml_ugly.toprettyxml()
file.write(xml_pretty.encode('utf-8'))
recording_flag = False
xlmns_namespaces = ""
with open(path_datastore + output_identifier.text + '.xml',
'r',
encoding="utf-8") as raw_datastore:
for line in raw_datastore:
# TODO, TURN THIS INTO A FUNCTION FOR THE TAGS XMLNS FIX
if "<iati-activity" in line:
if "iati-extra" in line:
line = line.replace(
'<iati-activity',
'<iati-activity xmlns:iati-extra="http://datastore.iatistandard.org/ns"'
)
recording_flag = True
if recording_flag == True:
if "</iati-activity" in line:
recording_flag = False
newf += "</iati-activity>"
else:
newf += line.strip() + "\n"
with open(path_activities + 'Output_{}.xml'.format(i + 1),
'r',
encoding="utf-8") as raw_list:
for line in raw_list:
newf2 += line.strip() + '\n'
with open(path_datastore + 'formatted-' + output_identifier.text +
'.xml',
'w',
encoding="utf-8") as formatted_datastore:
formatted_datastore.write(newf)
with open(path_activities + 'formatted-' + output_identifier.text +
'.xml',
'w',
encoding="utf-8") as formatted_list:
formatted_list.write(newf2)
if not os.stat(path_datastore + 'formatted-' + output_identifier.text +
'.xml').st_size == 0:
with open(path_differences + output_identifier.text + '.xml',
'w',
encoding="utf-8") as diff_file:
result = diffile.diff_files(path_activities + 'formatted-' +
output_identifier.text + '.xml',
path_datastore + 'formatted-' +
output_identifier.text + '.xml',
formatter=formatter,
diff_options={
'F': 1,
'ratio_mode': 'accurate'
})
if 'http://namespaces.shoobx.com/diff' in result:
result = result.replace(
'http://namespaces.shoobx.com/diff', '')
diff_file.write(result)
header = '''<!DOCTYPE html>
<html>
<head>
<link rel="stylesheet" href="../style.css">
</head>
<body>\n'''
footer = '''</body>
</html>
'''
with open(path_differences + output_identifier.text + '.html',
'w',
encoding="utf-8") as mockup_file_result:
mockup_file_result.write(header)
with open(path_differences + output_identifier.text + '.xml',
'r',
encoding="utf-8") as mockup_file:
for line in mockup_file:
if 'xmlns:diff=""' in line:
line = line.replace('xmlns:diff=""', '')
if '<' in line:
line = line.replace('<', '<')
if '>' in line:
line = line.replace('>', '>')
if 'diff:insert' in line:
counter = line.find('>')
if line[counter + 4] != line[-1]:
counter_closing = line.index('<', counter)
line = line[:counter +
4] + '<div class="DiffInsert"><pre>' + line[
counter + 4:
counter_closing] + '</pre></div>' + line[
counter_closing:]
line = line.replace('diff:insert=""', '')
if 'diff:del' in line:
counter = line.find('>')
if line[counter + 4] != line[-1]:
counter_closing = line.index('<', counter)
line = line[:counter +
4] + '<div class="DiffDel"><pre>' + line[
counter + 4:
counter_closing] + '</pre></div>' + line[
counter_closing:]
line = line.replace('diff:delete=""', '')
if 'diff:add-attr' in line:
# MAKE A LIST OF THE ATTRIBUTES IN ORDER TO SELECT THEM INDIVIDUALLY IN THE CSS LATER ON
attribute_selector = re.search(
'add-attr=\"(.+?)\"', line).group(1)
attribute_list = attribute_selector.split(";")
for attribute in attribute_list:
if str(attribute) in line:
try:
attribute_value = re.search(
str(attribute) + '=\"(.+?)\"',
line).group(1)
line = re.sub(
'(diff:add-attr=\".+?\")', '',
line)
line = line.replace(
attribute,
'<div class="DiffInsert"><pre>' +
attribute)
line = line.replace(
attribute_value + '"',
attribute_value + '"</pre></div>')
except AttributeError:
# It did not match the regex, likely because the attribute does not have any value and it is a value itself
line = re.sub(
'(diff:add-attr=\".+?\")', '',
line)
line = line.replace(
attribute,
'<div class="DiffInsert"><pre>' +
attribute + '</pre></div>')
#line = line.replace('diff:add-attr=', '<div class="DiffInsert"><pre>add-attr=')
#line = line + '</pre></div>'
mockup_file_result.write('<pre>' + line.strip() +
'</pre>' + '\n')
mockup_file_result.write(footer)
def compare_xmls(source_xml, target_xml):
diff = main.diff_files(source_xml,
target_xml,
formatter=formatting.XMLFormatter())
return diff
def xml_difference(original, to_compare):
diff = main.diff_files(original,
to_compare,
formatter=formatting.XMLFormatter())
return diff
def validate(filename_xsd, filename_xml, filename_xml_ground_truth):
"""
Validates the provided XML based on schema and checks whether the XML matches ground truth. Not matching ground
truth does not mean XML is invalid, but does mean manual inspection is required.
:param filename_xsd: The filename of the schema to validate against.
:param filename_xml: The filename of the XML to be validated.
:param filename_xml_ground_truth: The filename (or file-like object) to compare against.
"""
# Initialize Report
validation = {
"Well Formed": None,
"Schema Valid": None,
"Ground Truth": {
"Diff": None,
"Matched": False
},
"Error": {
"Message": None,
"Type": None
},
"File Names": {
"Ground Truth MDL": filename_xml_ground_truth,
"Transformed MDL": filename_xml
}
}
# parse xml
try:
xml_doc = etree.parse(filename_xml)
print('XML well formed, syntax ok.')
validation["Well Formed"] = True
# check for file IO error
except IOError:
msg = f'Invalid XML file: {filename_xml}'
write_error(msg, validation)
raise
# check for XML syntax errors
except etree.XMLSyntaxError as err:
msg = f'XML syntax Error: {err.error_log}'
write_error(msg, validation)
raise
except Exception as err:
msg = f'Unknown error parsing XML, exiting: {err}'
write_error(msg, validation)
raise
# validate against schema
try:
xmlschema_doc = etree.parse(filename_xsd)
xmlschema = etree.XMLSchema(xmlschema_doc)
xmlschema.assertValid(xml_doc)
print('XML valid, schema validation ok.')
validation["Schema Valid"] = True
# check for file IO error
except IOError:
msg = f'Invalid schema file: {filename_xsd}'
write_error(msg, validation)
raise
except etree.DocumentInvalid as err:
msg = f'Schema validation error: {err.error_log}'
write_error(msg, validation)
raise
except Exception as err:
msg = f'Unknown error validating XML against schema, exiting: {err}'
write_error(msg, validation)
raise
# diff ground truth
try:
xml_ground_truth = etree.parse(filename_xml_ground_truth)
xml_doc = parse_xml(filename_xml)
diff = main.diff_trees(xml_doc,
xml_ground_truth,
formatter=formatting.XMLFormatter())
if "diff:" not in diff:
print("Ground truth matches provided XML.")
validation["Ground Truth"]["Matched"] = True
else:
print("Ground truth not matched. Diff follows:")
print(diff)
validation["Ground Truth"]["Diff"] = diff
# check for file IO error
except IOError:
msg = f'Invalid ground truth file: {filename_xml_ground_truth}'
write_error(msg, validation)
raise
# check for XML syntax errors
except etree.XMLSyntaxError as err:
msg = f'XML syntax error in ground truth: {err.error_log}'
write_error(msg, validation)
raise
except Exception as err:
msg = f'Unknown error diffing XML, exiting: {err}'
write_error(msg, validation)
raise
write_report(validation)
