def parse_result(stdout, test_format=TestFormat.UNITY_FIXTURE_VERBOSE):
try:
results = TestResults(stdout, test_format)
except (ValueError, TypeError) as e:
raise ValueError(
'Error occurs when parsing the component unit test stdout to JUnit report: '
+ str(e))
group_name = results.tests()[0].group()
ComponentUTResult.results_list[group_name].append(results.to_junit())
with open(
os.path.join(os.getenv('LOG_PATH', ''),
'{}_XUNIT_RESULT.xml'.format(group_name)),
'w') as fw:
junit_xml.to_xml_report_file(
fw, ComponentUTResult.results_list[group_name])
if results.num_failed():
# raise exception if any case fails
err_msg = 'Failed Cases:\n'
for test_case in results.test_iter():
if test_case.result() == 'FAIL':
err_msg += '\t{}: {}'.format(test_case.name(),
test_case.message())
raise AssertionError(err_msg)
def handle_event(self, context: ExecutionContext,
event: events.ExecutionEvent) -> None:
if isinstance(event, events.Initialized):
self.start_time = event.start_time
if isinstance(event, events.AfterExecution):
test_case = TestCase(
f"{event.result.method} {event.result.path}",
elapsed_sec=event.elapsed_time,
allow_multiple_subelements=True,
)
if event.status == Status.failure:
checks = deduplicate_failures(event.result.checks)
for idx, check in enumerate(checks, 1):
# `check.message` is always not empty for events with `failure` status
test_case.add_failure_info(
message=f"{idx}. {check.message}")
if event.status == Status.error:
test_case.add_error_info(
message=event.result.errors[-1].exception,
output=event.result.errors[-1].exception_with_traceback)
self.test_cases.append(test_case)
if isinstance(event, events.Finished):
test_suites = [
TestSuite("schemathesis",
test_cases=self.test_cases,
hostname=platform.node())
]
to_xml_report_file(file_descriptor=self.file_handle,
test_suites=test_suites,
prettyprint=True)
def handle_composition(self, args: argparse.Namespace,
composition: mzcompose.Composition) -> None:
if args.workflow not in composition.workflows:
# Restart any dependencies whose definitions have changed. This is
# Docker Compose's default behavior for `up`, but not for `run`,
# which is a constant irritation that we paper over here. The trick,
# taken from Buildkite's Docker Compose plugin, is to run an `up`
# command that requests zero instances of the requested service.
if args.workflow:
composition.invoke(
"up",
"-d",
"--scale",
f"{args.workflow}=0",
args.workflow,
)
super().handle_composition(args, composition)
else:
# The user has specified a workflow rather than a service. Run the
# workflow instead of Docker Compose.
if args.unknown_args:
bad_arg = args.unknown_args[0]
elif args.unknown_subargs[0].startswith("-"):
bad_arg = args.unknown_subargs[0]
else:
bad_arg = None
if bad_arg:
raise UIError(
f"unknown option {bad_arg!r}",
hint=f"if {bad_arg!r} is a valid Docker Compose option, "
f"it can't be used when running {args.workflow!r}, because {args.workflow!r} "
"is a custom mzcompose workflow, not a Docker Compose service",
)
# Run the workflow inside of a test case so that we get some basic
# test analytics, even if the workflow doesn't define more granular
# test cases.
with composition.test_case(f"workflow-{args.workflow}"):
composition.workflow(args.workflow, *args.unknown_subargs[1:])
# Upload test report to Buildkite Test Analytics.
junit_suite = junit_xml.TestSuite(composition.name)
for (name, result) in composition.test_results.items():
test_case = junit_xml.TestCase(name, composition.name,
result.duration)
if result.error:
test_case.add_error_info(message=result.error)
junit_suite.test_cases.append(test_case)
junit_report = ci_util.junit_report_filename("mzcompose")
with junit_report.open("w") as f:
junit_xml.to_xml_report_file(f, [junit_suite])
ci_util.upload_junit_report("mzcompose", junit_report)
if any(result.error
for result in composition.test_results.values()):
raise UIError("at least one test case failed")
def teardown(self):
if len(self.failed_test):
test_cases = self.failed_test
else:
test_cases = list()
test_cases.append(TestCase(name='Fuzz test succeed',
status='Pass'))
if self.junit_report_path:
with open(self.junit_report_path, 'w') as report_file:
to_xml_report_file(report_file,
[TestSuite("API Fuzzer", test_cases)],
prettyprint=True)
super(ServerTarget, self).teardown() # pylint: disable=E1003
def serialize_and_read(test_suites,
to_file=False,
prettyprint=False,
encoding=None):
"""writes the test suite to an XML string and then re-reads it using minidom,
returning => (test suite element, list of test case elements)"""
try:
iter(test_suites)
except TypeError:
test_suites = [test_suites]
if to_file:
fd, filename = tempfile.mkstemp(text=True)
os.close(fd)
with codecs.open(filename, mode="w", encoding=encoding) as f:
to_xml_report_file(f,
test_suites,
prettyprint=prettyprint,
encoding=encoding)
print("Serialized XML to temp file [%s]" % filename)
xmldoc = minidom.parse(filename)
os.remove(filename)
else:
xml_string = to_xml_report_string(test_suites,
prettyprint=prettyprint,
encoding=encoding)
if PY2:
assert isinstance(xml_string, unicode) # noqa: F821
print("Serialized XML to string:\n%s" % xml_string)
if encoding:
xml_string = xml_string.encode(encoding)
xmldoc = minidom.parseString(xml_string)
def remove_blanks(node):
for x in node.childNodes:
if x.nodeType == minidom.Node.TEXT_NODE:
if x.nodeValue:
x.nodeValue = x.nodeValue.strip()
elif x.nodeType == minidom.Node.ELEMENT_NODE:
remove_blanks(x)
remove_blanks(xmldoc)
xmldoc.normalize()
ret = []
suites = xmldoc.getElementsByTagName("testsuites")[0]
for suite in suites.getElementsByTagName("testsuite"):
cases = suite.getElementsByTagName("testcase")
ret.append((suite, cases))
return ret
def output_test_summary(errors_total):
"""Prints summary of script output in form of junit-xml
Args:
errors_total (int): Total number of broken links
"""
if not os.path.isdir("test-summary"):
os.mkdir("test-summary")
with open("test-summary/junit-xml-report.xml", "w") as test_summary:
time_taken = time.time() - START_TIME
test_case = TestCase("Broken links checker", "License files",
time_taken)
if errors_total != 0:
test_case.add_failure_info(
f"{errors_total} broken links found",
f"Number of error links: {errors_total}\nNumber of unique"
f" broken links: {len(MAP_BROKEN_LINKS.keys())}",
)
ts = TestSuite("cc-link-checker", [test_case])
to_xml_report_file(test_summary, [ts])
def create_test_output(self, test_suites):
self.logger.info(to_xml_report_string(test_suites))
with open(self.output_file, 'w', encoding='utf-8') as f:
to_xml_report_file(f, test_suites)
self.logger.info('Junit xml output stored to %s', f.name)
except (IndexError, AttributeError):
main_log = None
case = junit_xml.TestCase(result.name,
classname=None,
elapsed_sec=duration_to_seconds(
result.duration),
stdout=main_log)
# Map tmt OUTCOME to JUnit states
if result.result == "error":
case.add_error_info(result.result)
elif result.result == "fail":
case.add_failure_info(result.result)
elif result.result == "info":
case.add_skipped_info(result.result)
elif result.result == "warn":
case.add_error_info(result.result)
# Passed state is the default
suite.test_cases.append(case)
f_path = self.opt("file", os.path.join(self.workdir, DEFAULT_NAME))
try:
with open(f_path, 'w') as fw:
if hasattr(junit_xml, 'to_xml_report_file'):
junit_xml.to_xml_report_file(fw, [suite])
else:
# For older junit-xml
junit_xml.TestSuite.to_file(fw, [suite])
self.info("output", f_path, 'yellow')
except Exception as error:
raise tmt.utils.ReportError(
f"Failed to write the output '{f_path}' ({error}).")
def output_report(cls, junit_file_path):
""" Output current test result to file. """
with open(os.path.join(junit_file_path, cls.JUNIT_FILE_NAME), 'w') as f:
junit_xml.to_xml_report_file(f, [cls.JUNIT_TEST_SUITE], prettyprint=False)
def run_notebook(
input_notebook,
add_nunit_attachment,
parameters=None,
kernel_name="ai-architecture-template",
root=".",
):
"""
Used to run a notebook in the correct directory.
Parameters
----------
:param input_notebook: Name of Notebook to Test
:param add_nunit_attachment:
:param parameters:
:param kernel_name: Jupyter Kernal
:param root:
"""
output_notebook = input_notebook.replace(".ipynb", notebook_output_ext)
try:
results = pm.execute_notebook(
os.path.join(root, input_notebook),
os.path.join(root, output_notebook),
parameters=parameters,
kernel_name=kernel_name,
)
for cell in results.cells:
if cell.cell_type == "code":
assert not cell.metadata.papermill.exception, "Error in Python Notebook"
finally:
with open(os.path.join(root, output_notebook)) as json_file:
data = json.load(json_file)
jupyter_output = nbformat.reads(json.dumps(data),
as_version=nbformat.NO_CONVERT)
export_md(
jupyter_output,
output_notebook,
add_nunit_attachment,
file_ext=".txt",
root=root,
)
regex = r"Deployed (.*) with name (.*). Took (.*) seconds."
with open(os.path.join(root, output_notebook)) as file:
data = file.read()
test_cases = []
for group in re.findall(regex, data):
test_cases.append(
TestCase(
name=group[0] + " creation",
classname=input_notebook,
elapsed_sec=float(group[2]),
status="Success",
))
test_suite = TestSuite("my test suite", test_cases)
with open("test-timing-output.xml", "w") as test_file:
to_xml_report_file(test_file, [test_suite], prettyprint=False)
