def lemmaPosMissingForm(inputFile, errorFile):
errorList = errors(errorFile)
tree = etree.parse(inputFile)
root = tree.getroot()
lemmas = root.findall('./sisu/lause/sone[@liik]')
fake_root = etree.Element(None) # Create 'fake' root node
pi = etree.PI('xml-stylesheet', 'type="text/xsl" href="liivike_tekst.xsl"') # Add desired processing instructions. Repeat as necessary.
pi.tail = "\n"
fake_root.append(pi)
fake_root.append(root) # Add real root as last child of fake root
tree = etree.ElementTree(fake_root)
for error in errorList:
for lemma in lemmas:
if "lemma" in lemma.attrib and "liik" in lemma.attrib and "vorm" not in lemma.attrib:
if lemma.attrib["lemma"] == error[0] and lemma.attrib["liik"] == error[1]:
print(lemma.attrib)
lemma.set("lemma", error[2]) # set to the right lemma
lemma.set("liik", error[3]) # set to the right pos
lemma.attrib["vorm"] = error[4] # add morph attribute
print(lemma.attrib)
outputFile = (re.sub(".xml", "", inputFile) + ".temp")
else:
continue
tree.write((re.sub(".xml", "", inputFile) + ".temp"), xml_declaration=True, encoding="utf-8")
def main():
feedxml_file = 'feed-' + GROUP_ID + '.xml'
logging.info(" ")
logging.info("===== START %s =====", GROUP_ID)
if not os.path.isfile(feedxml_file):
with open(feedxml_file, "w") as feed_file:
feed_file.write(
'<?xml version="1.0" encoding="UTF-8"?>\n'
'<?xml-stylesheet type="text/xsl" href="feed.xsl" ?>\n<feed>\n</feed>'
)
group_meta = get_group_meta()
tree = ET.parse(feedxml_file)
root = tree.getroot()
root.set("name", group_meta["name"])
root.set("description", group_meta["description"])
timestamps = list(
map(lambda p: int(p.text), root.findall("./post/timestamp")))
ids = list(map(lambda p: p.text, root.findall("./post/id")))
if len(timestamps) > 0:
last_timestamp = max(timestamps)
logging.info("Last downloaded timestamp: %s", last_timestamp)
else:
last_timestamp = None
feed = get_feed(
get_graph_url(
GROUP_ID + "/feed",
"updated_time,type,link,caption,created_time,description,"
"story,from,full_picture,message,picture,permalink_url,"
"object_id,attachments"), 0, last_timestamp)
for post in feed:
logging.info("GET POST %s OF TYPE %s", post["id"], post["type"])
if post["id"] not in ids:
root.insert(0, get_xmlelement(post))
indent(root)
fake_root = ET.Element(None)
stylesheet = ET.PI("xml-stylesheet", 'type="text/xsl" href="feed.xsl"')
stylesheet.tail = "\n"
fake_root.append(stylesheet)
# Add real root as last child of fake root
fake_root.append(root)
tree = ET.ElementTree(fake_root)
tree.write(feedxml_file, encoding="UTF-8", xml_declaration=True)
logging.info("Done")
def xmlboi(name, ra, ls, counter1, ident, item):
broute = ra.split("/")
troute = broute[0] + "/" + broute[1] + "/" + broute[2] + "/" + broute[
3] + "/" + broute[4] + "/" + broute[5] + "/"
os.chdir(troute)
f = open("summary_a" + str(counter1) + ".xml", 'w')
file = ET.Element('file')
fake_root = ET.Element(None)
pi = ET.PI("xml-stylesheet", "type='text/xsl' href='template.xslt'")
pi.tail = "\n"
fake_root.append(pi)
fake_root.append(file)
ET.SubElement(file, "file").text = str(name)
ET.SubElement(file, "relative").text = ("./" + item)
ET.SubElement(file, "line").text = str(ls)
ET.SubElement(file, "identifiers").text = str(ident)
(ET.ElementTree(element=fake_root)).write("summary_a" + str(counter1) +
".xml")
f.close()
def write_report(self, test_cases, test_classes):
# Start building final test report document
final_test_report = XMLTree.Element(
"testsuites") # Create root xml element
for string_class in test_classes: # add test suite elements as child of root
test_suite = XMLTree.Element('testsuite')
test_suite.set("name", string_class)
total_time = Decimal("0.0")
for case in test_cases:
case_class = case.get('classname')
test_time = Decimal(case.get('time'))
if case_class == string_class:
total_time = total_time + test_time
test_suite.append(case)
test_suite.set("time", str(total_time))
final_test_report.append(test_suite)
# Create 'fake' root node
output_header = XMLTree.Element(None)
# Create xml processing instruction for xml style sheet
pi = XMLTree.PI("xml-stylesheet",
"type='text/xsl' href='stylesheet.xsl'")
pi.tail = "\n"
output_header.append(pi)
output_header.append(final_test_report)
output_tree = XMLTree.ElementTree(output_header)
# Write the completed junit report
output_tree.write(self.destination_path,
encoding="UTF-8",
xml_declaration=True)
xml_string = minidom.parse(self.destination_path).toprettyxml()
xml_string = xml_string.replace(' at', ' at').replace(
'\n at',
' at').replace(' === Pre', ' === Pre').replace(
'=== Error', '=== Error')
with open(self.destination_path, "w") as f:
f.write(xml_string)
def handler(event, context):
print('Event: ')
print(json.dumps(event, sort_keys=False))
job_id = event["executionName"]
test_branch_job_identifier = event["testBranchJobIdentifier"]
timestamp_for_test_metrics = event["timestampForTestMetrics"]
sdb_domain = os.environ['SDBDOMAIN']
tmp_directory = "/tmp"
artifact_s3_bucket = os.environ['TASKRUNNERBUCKET']
artifact_s3_prefix = "test-runner/artifacts/"
report_s3_destination = "artifacts/" + test_branch_job_identifier + "/"
s3_destination_source = tmp_directory + "/upload/"
s3 = boto3.resource('s3')
clear_tmp()
os.makedirs("/tmp/artifacts")
os.makedirs("/tmp/upload")
# Read db for job metadata
page_iterator = get_sdb_paginator_for_test_results(sdb_domain, job_id)
# Read through db results for test ids, download junit reports
download_test_reports(page_iterator, artifact_s3_prefix,
artifact_s3_bucket, tmp_directory, s3)
# Read db for expired tests
expired_page_iterator = get_sdb_paginator_for_expired_tests(
sdb_domain, job_id)
# Grab testIds of expired tests, create test report if missing
expired_test_list = get_expired_test_list(expired_page_iterator,
tmp_directory)
# Get all of the downloaded junit files
junit_glob_pattern = tmp_directory + "/artifacts/*.xml"
file_list = glob.glob(junit_glob_pattern)
class_list = []
test_case_list = []
failed_test_list = []
skipped_test_list = []
# Open all files, copy testcase elements and class names for consolidated report
print("Number of files found: " + str(len(file_list)))
for item in file_list:
try:
root = XMLTree.parse(item)
except:
print("XML PARSE ERROR! FILE NAME: " + str(item))
# Skip because this file is screwed up, it will still get marked skipped by artifact-slurper
continue
# If file contains a failure or skip, add to failed/skipped list
test_case_id = os.path.basename(item).replace(".xml", "")
test_case_meta_data = test_case_id.split("-")
test_index = test_case_meta_data[0]
failed_list = root.findall("./testcase/failure")
skipped_list = root.findall("./testcase/skipped")
if len(failed_list) > 0:
failed_test_list.append(test_case_id)
elif len(skipped_list) > 0:
skipped_test_list.append(test_case_id)
# Grab all test cases
for testcase in root.findall(
"./testcase"): # xpath expression strips parent elements
classname = testcase.get(
"classname"
) # reads the xml attribute 'classname' from testcase element
testcase.set("testid", test_case_id)
testcase.set("testindex", test_index)
testcase.set("jobId", job_id)
testcase.set("isfailed", "false")
testcase.set("isskipped", "false")
testcase.set("ispassed", "false")
testcase.set("isexpired", "false")
if test_case_id in failed_test_list:
testcase.set("isfailed", "true")
elif test_case_id in skipped_test_list:
testcase.set("isskipped", "true")
elif test_case_id in expired_test_list:
failure = XMLTree.Element('failure')
failure.set(
"message",
"Exceeded Timeout in AWS Lambda, see cloudwatch log!")
failure.set("type", "AWS_TIMEOUT")
testcase.set("isexpired", "true")
testcase.set("isfailed", "true")
testcase.append(failure)
else:
testcase.set("ispassed", "true")
class_list.append(classname) # Class name saved as string
test_case_list.append(
testcase) # Test case saved as raw XML element
class_list = list(
set(class_list)
) # Python hack to remove duplicates, these become parent testsuite elements
# Start building final test report document
final_test_report = XMLTree.Element(
"testsuites") # Create root xml element
for string_class in class_list: # add test suite elements as child of root
testsuite = XMLTree.Element('testsuite')
testsuite.set("name", string_class)
total_time = Decimal("0.0")
for case in test_case_list:
caseclass = case.get('classname')
testtime = Decimal(case.get('time'))
if caseclass == string_class:
total_time = total_time + testtime
testsuite.append(case)
testsuite.set("time", str(total_time))
final_test_report.append(testsuite)
output_file = tmp_directory + "/upload/" + timestamp_for_test_metrics + ".xml"
# Create 'fake' root node
output_header = XMLTree.Element(None)
# Create xml processing instruction for xml style sheet
pi = XMLTree.PI("xml-stylesheet", "type='text/xsl' href='stylesheet.xsl'")
pi.tail = "\n"
output_header.append(pi)
output_header.append(final_test_report)
output_tree = XMLTree.ElementTree(output_header)
# Write the completed junit report
output_tree.write(output_file, encoding="UTF-8", xml_declaration=True)
xml_str = minidom.parse(output_file).toprettyxml()
xml_str = xml_str.replace(' at', ' at').replace(
'\n at',
' at').replace(' === Pre', ' === Pre').replace(
'=== Error', '=== Error')
with open(output_file, "w") as f:
f.write(xml_str)
# Define a wildcard expression to grab paths of files to be uploaded
source_glob_pattern = s3_destination_source + "*"
files_to_upload = glob.glob(source_glob_pattern)
for file_to_upload in files_to_upload:
destination_key = report_s3_destination + os.path.basename(
file_to_upload)
content_type = "binary/octet-stream"
if ".xml" in file_to_upload:
content_type = "text/xml"
logger.info("Uploading " + file_to_upload + " to S3 path " +
artifact_s3_bucket + "/" + destination_key)
s3.Object(artifact_s3_bucket,
destination_key).put(Body=open(file_to_upload, 'rb'),
ContentType=content_type,
ContentDisposition='inline')
# Define and upload the xml style sheet
# TODO: https://stackoverflow.com/questions/6028000/how-to-read-a-static-file-from-inside-a-python-package
xsl_sheet = "/var/task/functions/stylesheet.xsl"
xsl_destination = report_s3_destination + "stylesheet.xsl"
logger.info("Uploading stylesheet.xsl to S3 path " + artifact_s3_bucket +
"/" + xsl_destination)
s3.Object(artifact_s3_bucket,
xsl_destination).put(Body=open(xsl_sheet, 'rb'),
ContentType='text/xml',
ContentDisposition='inline')
# Create url for final test report and return
return_url = report_s3_destination + timestamp_for_test_metrics + ".xml"
return return_url
def _handle_pi(self, target, data):
return ET.PI(self._fixtext(target), self._fixtext(data))
