def output_run_results_junit_xml(passing_tests: List[TestRunResult],
failing_tests: List[TestRunResult],
junit_dest: TextIO,
junit_merged_dest: TextIO):
'''Write results to JUnit XML
Two versions are produced: a normal version and a merged version. In the
normal version there is a test suite per unique test name with a different
test case per seed run. In the merged version there is a single test case
under the test suite with information for the individual runs merged
together. This is to aid use of the Azure Pipelines JUnit dashboard, which
doesn't neatly handle the test suite/test case hierarchy
'''
all_tests = passing_tests + failing_tests
test_suite_info = {}
for trr in all_tests:
# test_case_info contains a tuple per unique test name. The first
# element is a list of junit_xml.TestCase, one per test run with that
# name. The other merges together all of the test outputs to produce
# the merged output.
unmerged, merged = \
test_suite_info.setdefault(trr.name, ([], {'stdout': '',
'failures': ''}))
result_text = gen_test_run_result_text(trr)
# Create a test case for the TestRunResult. stdout holds the text
# describing the run. Add the same text to failures if the test failed.
test_case = junit_xml.TestCase(f'{trr.name}.{trr.seed}')
test_case.stdout = result_text
merged['stdout'] += result_text + '\n'
if not trr.passed:
test_case.add_failure_info(output=result_text)
merged['failures'] += result_text
unmerged.append(test_case)
# Output the normal JUnit XML
test_suites = [
junit_xml.TestSuite(name, test_cases)
for name, (test_cases, _) in test_suite_info.items()
]
junit_dest.write(junit_xml.to_xml_report_string(test_suites))
# Output the merged version of the JUnit XML
merged_test_suites = []
for name, (_, merged_test_info) in test_suite_info.items():
test_case = junit_xml.TestCase(name)
test_case.stdout = merged_test_info['stdout']
test_case.add_failure_info(output=merged_test_info['failures'])
merged_test_suites.append(junit_xml.TestSuite(name, [test_case]))
junit_merged_dest.write(junit_xml.to_xml_report_string(merged_test_suites))
def print_matches(self, matches, rules=None, filenames=None):
"""Output all the matches"""
if not rules:
return None
test_cases = []
for rule in rules.all_rules:
if not rules.is_rule_enabled(rule):
if not rule.id:
continue
test_case = TestCase(
name='{0} {1}'.format(rule.id, rule.shortdesc))
if rule.experimental:
test_case.add_skipped_info(
message='Experimental rule - not enabled')
else:
test_case.add_skipped_info(message='Ignored rule')
test_cases.append(test_case)
else:
test_case = TestCase(name='{0} {1}'.format(
rule.id, rule.shortdesc),
allow_multiple_subelements=True,
url=rule.source_url)
for match in matches:
if match.rule.id == rule.id:
test_case.add_failure_info(
message=self._failure_format(match),
failure_type=match.message)
test_cases.append(test_case)
test_suite = TestSuite('CloudFormation Lint', test_cases)
return to_xml_report_string([test_suite], prettyprint=True)
def format_baseline_for_junit_xml(baseline):
"""
:type baseline: dict
:rtype: str
"""
all_secrets = {}
for filename, secret_list in baseline['results'].items():
for secret in secret_list:
test_case = junit_xml.TestCase(
name="{}:{}".format(filename, secret["line_number"]))
test_case.add_failure_info(
message="Found secret of type {} on line {} in file {}".format(
secret["type"], secret["line_number"], filename),
failure_type=secret["type"])
if secret["type"] in all_secrets:
all_secrets[secret["type"]].append(test_case)
else:
all_secrets[secret["type"]] = [test_case]
test_suits = map(
lambda secret: junit_xml.TestSuite(name=secret[0],
test_cases=secret[1]),
all_secrets.items())
return junit_xml.to_xml_report_string(test_suits)
def write_junitxml(output_junitxml, results):
"""Write output file as JUnitXML format"""
if not JUNIT_XML_FOUND:
log = logging.getLogger(__name__ + ".write_junitxml")
log.warning('junitxml output disabled: the `junit_xml` python module '
'is missing.')
return
test_cases = []
duration_re = re.compile('([0-9]+):([0-9]+):([0-9]+).([0-9]+)')
for vitem in results:
if vitem.get('Validations'):
parsed_duration = 0
test_duration = vitem.get('Duration', '')
matched_duration = duration_re.match(test_duration)
if matched_duration:
parsed_duration = (int(matched_duration[1])*3600
+ int(matched_duration[2])*60
+ int(matched_duration[3])
+ float('0.{}'.format(matched_duration[4])))
test_stdout = vitem.get('Status_by_Host', '')
test_case = TestCase('validations', vitem['Validations'],
parsed_duration, test_stdout)
if vitem['Status'] == 'FAILED':
test_case.add_failure_info('FAILED')
test_cases.append(test_case)
ts = TestSuite("Validations", test_cases)
with open(output_junitxml, 'w') as output:
output.write(to_xml_report_string([ts]))
def create_xml_report(failures, dockerfile_path):
"""Make a full XML report file."""
test_case = TestCase(
"Lint " + dockerfile_path,
classname="dockerlint.main",
)
for f in failures:
test_case.add_failure_info(message=f.__str__())
ts = TestSuite("dockerlint", test_cases=[test_case])
return to_xml_report_string([ts])
def serialize_and_read(test_suites,
to_file=False,
prettyprint=False,
encoding=None):
"""writes the test suite to an XML string and then re-reads it using minidom,
returning => (test suite element, list of test case elements)"""
try:
iter(test_suites)
except TypeError:
test_suites = [test_suites]
if to_file:
fd, filename = tempfile.mkstemp(text=True)
os.close(fd)
with codecs.open(filename, mode="w", encoding=encoding) as f:
to_xml_report_file(f,
test_suites,
prettyprint=prettyprint,
encoding=encoding)
print("Serialized XML to temp file [%s]" % filename)
xmldoc = minidom.parse(filename)
os.remove(filename)
else:
xml_string = to_xml_report_string(test_suites,
prettyprint=prettyprint,
encoding=encoding)
if PY2:
assert isinstance(xml_string, unicode) # noqa: F821
print("Serialized XML to string:\n%s" % xml_string)
if encoding:
xml_string = xml_string.encode(encoding)
xmldoc = minidom.parseString(xml_string)
def remove_blanks(node):
for x in node.childNodes:
if x.nodeType == minidom.Node.TEXT_NODE:
if x.nodeValue:
x.nodeValue = x.nodeValue.strip()
elif x.nodeType == minidom.Node.ELEMENT_NODE:
remove_blanks(x)
remove_blanks(xmldoc)
xmldoc.normalize()
ret = []
suites = xmldoc.getElementsByTagName("testsuites")[0]
for suite in suites.getElementsByTagName("testsuite"):
cases = suite.getElementsByTagName("testcase")
ret.append((suite, cases))
return ret
def _generate_report(self):
""" generate a TestSuite report from the collected TaskData and HostData """
test_cases = []
for task_uuid, task_data in self._task_data.items():
if task_data.action == 'setup' and self._include_setup_tasks_in_report == 'false':
continue
for host_uuid, host_data in task_data.host_data.items():
test_cases.append(self._build_test_case(task_data, host_data))
test_suite = TestSuite(self._playbook_name, test_cases)
report = to_xml_report_string([test_suite])
output_file = os.path.join(self._output_dir, '%s-%s.xml' % (self._playbook_name, time.time()))
with open(output_file, 'wb') as xml:
xml.write(to_bytes(report, errors='surrogate_or_strict'))
def test_to_xml_string():
test_suites = [
Suite(name="suite1", test_cases=[Case(name="Test1")]),
Suite(name="suite2", test_cases=[Case(name="Test2")]),
]
xml_string = to_xml_report_string(test_suites)
if PY2:
assert isinstance(xml_string, unicode) # pylint: disable=undefined-variable
expected_xml_string = textwrap.dedent("""
<?xml version="1.0" ?>
<testsuites disabled="0" errors="0" failures="0" tests="2" time="0.0">
\t<testsuite disabled="0" errors="0" failures="0" name="suite1" skipped="0" tests="1" time="0">
\t\t<testcase name="Test1"/>
\t</testsuite>
\t<testsuite disabled="0" errors="0" failures="0" name="suite2" skipped="0" tests="1" time="0">
\t\t<testcase name="Test2"/>
\t</testsuite>
</testsuites>
""".strip("\n"))
assert xml_string == expected_xml_string
def process_test_results(test_results, output_folder="./", test_labels=None):
"""
This function writes test results to a file, displays failed tests to stdout,
and throws an exception if there are test failures.
"""
label_string = ""
if test_labels:
label_string = "with labels: {}".format(str(test_labels))
test_suites = [
TestSuite("Open-CE tests for {} {}".format(feedstock, label_string),
test_results[feedstock]) for feedstock in test_results
]
with open(os.path.join(output_folder, utils.DEFAULT_TEST_RESULT_FILE),
'w') as outfile:
outfile.write(to_xml_report_string(test_suites))
failed_tests = [
x for key in test_results for x in test_results[key] if x.is_failure()
]
if failed_tests:
raise OpenCEError(
Error.FAILED_TESTS, len(failed_tests),
str([failed_test.name for failed_test in failed_tests]))
log.info("All tests passed!")
def _export(self, suite: TestSuite, force=False) -> None:
"""
Export test suite to JUnit xml file
:param suite: TestSuite to export
:return: None
"""
if not suite or len(suite.test_cases) == 0:
return
if not self._has_uncollected_fixtures or force:
values = self._get_parametrize_as_str()
path = self._report_dir.joinpath(
self.get_report_file_name(
suite_name=suite.name,
args=values,
custom_filename=self._custom_filename))
xml_string = to_xml_report_string([suite])
os.makedirs(self._report_dir, exist_ok=True)
with open(path, "w") as f:
f.write(xml_string)
self.clear_cases()
def export_service_logs_to_junit_suites(cls, source_dir: Path,
report_dir: Path):
suites = list()
for file in source_dir.glob("logs_assisted-service*.log"):
suite_name = Path(file).stem.replace("logs_", "")
log.info(f"Creating test suite from {suite_name}.log")
test_cases = cls.get_failure_cases(file, suite_name)
timestamp = test_cases[0].timestamp if test_cases else None
suites.append(
TestSuite(name=suite_name,
test_cases=test_cases,
timestamp=timestamp))
log.info(f"Generating xml file for {len(suites)} suites")
xml_report = to_xml_report_string(suites)
with open(
report_dir.joinpath(
f"junit_log_parser_{str(uuid.uuid4())[:8]}.xml"),
"w") as f:
log.info(
f"Exporting {len(suites)} suites xml-report with {len(xml_report)} characters to {f.name}"
)
f.write(xml_report)
def collect(self,
entries: List[Dict[str, str]],
suite_name: str,
report_dir: Optional[Path] = None,
xml_suffix: str = "") -> str:
report_dir = Utils.get_report_dir(report_dir)
test_cases = list()
for entry in entries:
test_case = self._get_test_case(
entry, entry.get(self._format.severity_key, None))
if test_case is not None:
test_cases.append(test_case)
report_dir.mkdir(exist_ok=True)
xml_report = to_xml_report_string(test_suites=[
TestSuite(name=suite_name,
test_cases=test_cases,
timestamp=self._get_suite_timestamp(test_cases))
])
file_name = f"{self._report_prefix}_{suite_name}{f'_{xml_suffix}' if xml_suffix else ''}.xml"
with open(report_dir.joinpath(file_name), "w") as f:
f.write(xml_report)
return f.name
def export_service_events_to_junit_suite(
cls,
source_dir: Path,
report_dir: Path,
events_file_name="k8s_events.json"):
with open(source_dir.joinpath(events_file_name)) as f:
events_data = json.load(f)
log.info(
f"Creating test suite from service events json file - {events_file_name}"
)
test_cases = cls.get_event_test_cases(events_data)
log.info(f"Generating events xml file")
xml_report = to_xml_report_string(
test_suites=[TestSuite(name="EVENTS", test_cases=test_cases)])
with open(
report_dir.joinpath(
f"junit_events_parser_{str(uuid.uuid4())[:8]}.xml"),
"w") as f:
log.info(
f"Exporting events xml-report with {len(test_cases)} events to {f.name}"
)
f.write(xml_report)
def create_test_output(self, test_suites):
self.logger.info(to_xml_report_string(test_suites))
with open(self.output_file, 'w', encoding='utf-8') as f:
to_xml_report_file(f, test_suites)
self.logger.info('Junit xml output stored to %s', f.name)
result = unittest.TestResult()
test.run(result)
case = junit.TestCase(name=str(test))
for error in result.errors:
case.add_error_info(message=error[1])
err_cnt += 1
for failure in result.failures:
case.add_failure_info(message=failure[1])
fail_cnt += 1
for skip in result.skipped:
case.add_skipped_info(message=skip[1])
cases.append(case)
ts = junit.TestSuite(name="test", test_cases=cases)
results = junit.to_xml_report_string([ts], encoding="utf-8")
if opt.output is None:
print(results)
else:
with open(opt.output, "w") as f:
f.write(results)
os.sys.exit(err_cnt | fail_cnt)
)
if case.result == TestCase.RESULT_FAIL:
logs = None
# TODO: is this of any use? (yaml inside xml!)
if (case.start_log_line is not None
and case.end_log_line is not None):
logs = logs_instance.read(self.get_object(),
case.start_log_line,
case.end_log_line)
tc.add_error_info("failed", output=logs)
elif case.result == TestCase.RESULT_SKIP:
tc.add_skipped_info("skipped")
cases.append(tc)
suites.append(junit_xml.TestSuite(suite.name, test_cases=cases))
data = junit_xml.to_xml_report_string(suites, encoding="utf-8")
response = HttpResponse(data, content_type="application/xml")
response["Content-Disposition"] = ("attachment; filename=job_%d.xml" %
self.get_object().id)
return response
@detail_route(methods=["get"], suffix="logs")
def logs(self, request, **kwargs):
start = safe_str2int(request.query_params.get("start", 0))
end = safe_str2int(request.query_params.get("end", None))
try:
if start == 0 and end is None:
data = logs_instance.open(self.get_object())
response = FileResponse(data, content_type="application/yaml")
else:
data = logs_instance.read(self.get_object(), start, end)
def get_junit_xml_string(ts: List[TestSuite]) -> str:
return to_xml_report_string(ts)
def test_to_xml_string_test_suites_not_a_list():
test_suites = Suite("suite1", [Case("Test1")])
with pytest.raises(TypeError) as excinfo:
to_xml_report_string(test_suites)
assert str(excinfo.value) == "test_suites must be a list of test suites"
