def output_run_results_junit_xml(passing_tests: List[TestRunResult],
failing_tests: List[TestRunResult],
junit_dest: TextIO,
junit_merged_dest: TextIO):
'''Write results to JUnit XML
Two versions are produced: a normal version and a merged version. In the
normal version there is a test suite per unique test name with a different
test case per seed run. In the merged version there is a single test case
under the test suite with information for the individual runs merged
together. This is to aid use of the Azure Pipelines JUnit dashboard, which
doesn't neatly handle the test suite/test case hierarchy
'''
all_tests = passing_tests + failing_tests
test_suite_info = {}
for trr in all_tests:
# test_case_info contains a tuple per unique test name. The first
# element is a list of junit_xml.TestCase, one per test run with that
# name. The other merges together all of the test outputs to produce
# the merged output.
unmerged, merged = \
test_suite_info.setdefault(trr.name, ([], {'stdout': '',
'failures': ''}))
result_text = gen_test_run_result_text(trr)
# Create a test case for the TestRunResult. stdout holds the text
# describing the run. Add the same text to failures if the test failed.
test_case = junit_xml.TestCase(f'{trr.name}.{trr.seed}')
test_case.stdout = result_text
merged['stdout'] += result_text + '\n'
if not trr.passed:
test_case.add_failure_info(output=result_text)
merged['failures'] += result_text
unmerged.append(test_case)
# Output the normal JUnit XML
test_suites = [
junit_xml.TestSuite(name, test_cases)
for name, (test_cases, _) in test_suite_info.items()
]
junit_dest.write(junit_xml.to_xml_report_string(test_suites))
# Output the merged version of the JUnit XML
merged_test_suites = []
for name, (_, merged_test_info) in test_suite_info.items():
test_case = junit_xml.TestCase(name)
test_case.stdout = merged_test_info['stdout']
test_case.add_failure_info(output=merged_test_info['failures'])
merged_test_suites.append(junit_xml.TestSuite(name, [test_case]))
junit_merged_dest.write(junit_xml.to_xml_report_string(merged_test_suites))
def create_test_case(self, test):
# type: (Dict[Text, Any]) -> junit_xml.TestCase
doc = test.get(u'doc', 'N/A').strip()
return junit_xml.TestCase(doc,
elapsed_sec=self.duration,
stdout=self.standard_output,
stderr=self.error_output)
def format_baseline_for_junit_xml(baseline):
"""
:type baseline: dict
:rtype: str
"""
all_secrets = {}
for filename, secret_list in baseline['results'].items():
for secret in secret_list:
test_case = junit_xml.TestCase(
name="{}:{}".format(filename, secret["line_number"]))
test_case.add_failure_info(
message="Found secret of type {} on line {} in file {}".format(
secret["type"], secret["line_number"], filename),
failure_type=secret["type"])
if secret["type"] in all_secrets:
all_secrets[secret["type"]].append(test_case)
else:
all_secrets[secret["type"]] = [test_case]
test_suits = map(
lambda secret: junit_xml.TestSuite(name=secret[0],
test_cases=secret[1]),
all_secrets.items())
return junit_xml.to_xml_report_string(test_suits)
def create_junit_output(tests):
""" Creates an output file for generating a junit test report
args:
tests: All tests completed + skipped
returns:
boolean: False if file generation failed - NOT YET
"""
# Populate junit objects for all performed tests
junit_tests = []
# Integration tests
for test in tests:
junit_object = jx.TestCase(test.name_, classname="IncludeOS.{}.{}".format(test.type_, test.category_))
# If test is skipped add skipped info
if test.skip_:
junit_object.add_skipped_info(message = test.skip_reason_, output = test.skip_reason_)
elif test.proc_.returncode is not 0:
junit_object.add_failure_info(output = test.output_[0])
else:
junit_object.stdout = test.output_[0]
junit_object.stderr = test.output_[1]
# Add to list of all test objects
junit_tests.append(junit_object)
# Stress and misc tests
ts = jx.TestSuite("IncludeOS tests", junit_tests)
with open('output.xml', 'w') as f:
jx.TestSuite.to_file(f, [ts], prettyprint=False)
def to_junit(
self,
suite_name="all_tests",
):
"""
Convert the tests to JUnit XML.
Returns a junit_xml.TestSuite containing all of the test cases. One test suite will be
generated with the name given in suite_name. Unity Fixture test groups are mapped to the
classname attribute of test cases; for basic Unity output there will be one class named
"default".
Optional arguments:
suite_name -- The name to use for the "name" and "package" attributes of the testsuite element.
Sample output:
<testsuite disabled="0" errors="0" failures="1" name="[suite_name]" package="[suite_name]" skipped="0" tests="8" time="0">
<testcase classname="test_group_1" name="group_1_test" />
<testcase classname="test_group_2" name="group_2_test" />
</testsuite>
"""
test_case_list = []
for test in self._tests:
if test.result() == "PASS":
test_case_list.append(
junit_xml.TestCase(name=test.name(),
classname=test.group()))
else:
junit_tc = junit_xml.TestCase(
name=test.name(),
classname=test.group(),
file=test.file(),
line=test.line(),
)
if test.result() == "FAIL":
junit_tc.add_failure_info(message=test.message(),
output=test.full_line())
elif test.result() == "IGNORE":
junit_tc.add_skipped_info(message=test.message(),
output=test.full_line())
test_case_list.append(junit_tc)
return junit_xml.TestSuite(name=suite_name,
package=suite_name,
test_cases=test_case_list)
def _mkreport_junit(self, _tc, kws, header, output, tag_info,
reproduction):
for hook in self.junit_hooks:
hook(self, _tc, kws, output)
jtc = junit_xml.TestCase(self.junit_name % kws,
classname=self.junit_classname % kws,
elapsed_sec=123.456,
stdout=header + tag_info + reproduction,
stderr=None)
# FIXME: nail down the exception
# <error/failure/blockage/skipped/or cause to put that only in
# the messagee and let's put the whole output always in
# stdout, with the rest of the info on stderr
msg_tag = kws['msg_tag']
if msg_tag == "FAIL":
jtc.add_failure_info(message="Failed", output=output)
elif msg_tag == "ERRR":
jtc.add_error_info(message="Error", output=output)
elif msg_tag == "BLCK":
jtc.add_error_info(message="Infrastructure", output=output)
elif msg_tag == "SKIP":
if self.junit_report_skip:
jtc.add_skipped_info(message="Skipped", output=output)
else:
jtc.add_skipped_info(message="Skipped")
jtc.stdout = None
jtc.stderr = None
elif msg_tag == "PASS":
if self.junit_report_pass:
jtc.stderr = output
elif self.junit_report_pass == None:
# we don want *anything*
jtc.stderr = None
jtc.stdout = None
else: # False
jtc.stderr = "<inclusion of output disabled by " \
"configuration setting of " \
"tcfl.report.junit_report_pass>"
jtc.stdout = "<inclusion of output disabled by " \
"configuration setting of " \
"tcfl.report.junit_report_pass>"
# Write the JUNIT to a pickle file, as we'll join it later
# with the rest in _finalize. We can't put it in a
# global because this testcase might be running in a separate
# thread or process. later == when the global testcase
# reporter (tcfl.tc.tc_global) emits a COMPLETION message,
# then we call _finalize()
domain = commonl.file_name_make_safe(self.junit_domain % kws)
# use the core keywords, so it is not modified
tc_hash = _tc.kws['tc_hash']
# Note we store it in the common
pathname = os.path.join(tcfl.tc.tc_c.tmpdir, "junit", domain)
commonl.makedirs_p(pathname)
with open(os.path.join(pathname, tc_hash + ".pickle"), "w") as picklef:
cPickle.dump(jtc, picklef, protocol=2)
def create_test_case(self, test):
# type: (Dict[Text, Any]) -> junit_xml.TestCase
doc = test.get(u'doc', 'N/A').strip()
case = junit_xml.TestCase(doc,
elapsed_sec=self.duration,
stdout=self.standard_output,
stderr=self.error_output)
if self.return_code > 0:
case.failure_message = self.message
def handle_composition(self, args: argparse.Namespace,
composition: mzcompose.Composition) -> None:
if args.workflow not in composition.workflows:
# Restart any dependencies whose definitions have changed. This is
# Docker Compose's default behavior for `up`, but not for `run`,
# which is a constant irritation that we paper over here. The trick,
# taken from Buildkite's Docker Compose plugin, is to run an `up`
# command that requests zero instances of the requested service.
if args.workflow:
composition.invoke(
"up",
"-d",
"--scale",
f"{args.workflow}=0",
args.workflow,
)
super().handle_composition(args, composition)
else:
# The user has specified a workflow rather than a service. Run the
# workflow instead of Docker Compose.
if args.unknown_args:
bad_arg = args.unknown_args[0]
elif args.unknown_subargs[0].startswith("-"):
bad_arg = args.unknown_subargs[0]
else:
bad_arg = None
if bad_arg:
raise UIError(
f"unknown option {bad_arg!r}",
hint=f"if {bad_arg!r} is a valid Docker Compose option, "
f"it can't be used when running {args.workflow!r}, because {args.workflow!r} "
"is a custom mzcompose workflow, not a Docker Compose service",
)
# Run the workflow inside of a test case so that we get some basic
# test analytics, even if the workflow doesn't define more granular
# test cases.
with composition.test_case(f"workflow-{args.workflow}"):
composition.workflow(args.workflow, *args.unknown_subargs[1:])
# Upload test report to Buildkite Test Analytics.
junit_suite = junit_xml.TestSuite(composition.name)
for (name, result) in composition.test_results.items():
test_case = junit_xml.TestCase(name, composition.name,
result.duration)
if result.error:
test_case.add_error_info(message=result.error)
junit_suite.test_cases.append(test_case)
junit_report = ci_util.junit_report_filename("mzcompose")
with junit_report.open("w") as f:
junit_xml.to_xml_report_file(f, [junit_suite])
ci_util.upload_junit_report("mzcompose", junit_report)
if any(result.error
for result in composition.test_results.values()):
raise UIError("at least one test case failed")
def testcase_to_junit(
toplevel: str,
i: int,
cls_path: str,
testcase: TestcaseExecution,
is_toplevel: bool = False,
) -> typing.Tuple[int, typing.List[junit_xml.TestCase]]:
"""Convert a testcase to junit testcases."""
testcases = []
my_cls_path = cls_path if is_toplevel else f"{cls_path} -> {testcase.name}"
tc = junit_xml.TestCase(
f"99999 - Summary {testcase.name}",
classname=f"{toplevel}.{i:05} - {my_cls_path}",
)
if not testcase.success:
if testcase.exc is not None:
tc.add_error_info(f'Testcase failed with "{testcase.exc}"',
testcase.trace)
else:
tc.add_error_info(
f"Testcase failed because of sub testcase failure")
testcases.append(tc)
old_i = i
i += 1
for step_id, step in enumerate(testcase.sub_steps):
if isinstance(step, TestcaseExecution):
tc = junit_xml.TestCase(
f"{step_id:05} - Testcase: {step.name}",
classname=f"{toplevel}.{old_i:05} - {my_cls_path}",
)
testcases.append(tc)
i_new, testcases_new = testcase_to_junit(toplevel, i, my_cls_path,
step)
i = i_new
testcases += testcases_new
elif isinstance(step, ShellStep):
tc = junit_xml.TestCase(
f"{step_id:05} - Shell: {step.command}",
classname=f"{toplevel}.{old_i:05} - {my_cls_path}",
stdout=step.output,
)
testcases.append(tc)
return i, testcases
def emit_junit_xml(cluster_launch_attempts, frameworks):
"""Write out all the test actions failures to a junit file for jenkins or
similar"""
if not ok():
return
import junit_xml
launch_fake_testcases = []
for launch_attempt in cluster_launch_attempts:
attempt_duration = launch_attempt.end_time - launch_attempt.start_time
fake_test = junit_xml.TestCase(launch_attempt.name,
elapsed_sec=attempt_duration)
if launch_attempt.launch_succeeded:
fake_test.stdout = "Launch worked"
else:
fake_test.add_failure_info("Launch failed")
launch_fake_testcases.append(fake_test)
launch_suite = junit_xml.TestSuite("Cluster launches",
launch_fake_testcases)
fake_suites = []
fake_suites.append(launch_suite)
for framework in frameworks:
framework_testcases = []
for action_name, action in framework.actions.items():
action_duration = action['finish'] - action['start']
fake_test = junit_xml.TestCase(action_name,
elapsed_sec=action_duration,
stdout=action['stdout'],
stderr=action['stderr'])
if not action['ok']:
message = action['error_message']
if not message:
message = "%s failed" % action_name
fake_test.add_failure_info(message, action['error_output'])
framework_testcases.append(fake_test)
framework_suite = junit_xml.TestSuite("%s actions" % framework.name,
framework_testcases)
fake_suites.append(framework_suite)
with open("junit_testpy.xml", "w") as f:
junit_xml.TestSuite.to_file(f, fake_suites)
def create_junit_xml_file(projects, xml_file_path):
"""Create JUnit XML report file that can be used in Jenkins."""
synced_projects = []
for project in projects:
for k, v in project.items():
tc = junit_xml.TestCase(k)
tc.add_error_info(message=v[1])
synced_projects.append(tc)
ts = junit_xml.TestSuite("Sync", synced_projects)
with open(xml_file_path, 'w') as f:
junit_xml.TestSuite.to_file(f, [ts])
def create_test_case(cls, name):
"""
Extend ``junit_xml.TestCase`` with:
1. save create test case so it can be get by ``get_current_test_case``
2. log create timestamp, so ``elapsed_sec`` can be auto updated in ``test_case_finish``.
:param name: test case name
:return: instance of ``junit_xml.TestCase``
"""
# set stdout to empty string, so we can always append string to stdout.
# It won't affect output logic. If stdout is empty, it won't be put to report.
test_case = junit_xml.TestCase(name, stdout="")
cls.JUNIT_CURRENT_TEST_CASE = test_case
cls._TEST_CASE_CREATED_TS = time.time()
return test_case
def _end_test(
self, example, type='Passed', err_msg='', out_msg='', last=False
):
if not last:
last = self.current_startedAt
self.current_startedAt = time.time()
self.junitxml_tests[self.current_group]['tests'].append(
junit_xml.TestCase(
name=example.name,
classname=self.current_group,
status=type,
elapsed_sec=self.current_startedAt - last,
stderr=err_msg,
stdout=out_msg
)
)
def create_test_case(self, test):
# type: (Dict[Text, Any]) -> junit_xml.TestCase
doc = test.get(u"doc", "N/A").strip()
if test.get("tags"):
category = ", ".join(test["tags"])
else:
category = REQUIRED
short_name = test.get(u"short_name")
case = junit_xml.TestCase(
doc,
elapsed_sec=self.duration,
file=short_name,
category=category,
stdout=self.standard_output,
stderr=self.error_output,
)
if self.return_code > 0:
case.failure_message = self.message
def addSubTest(self, test, subtest, err):
"""Called at the end of a subtest.
'err' is None if the subtest ended successfully, otherwise it's a
tuple of values as returned by sys.exc_info().
"""
self.current_case = junit.TestCase(name=str(subtest))
self.cases.append(self.current_case)
if err is not None:
if issubclass(err[0], test.failureException):
self.current_case.add_failure_info(
message=self._exc_info_to_string(err, test))
else:
self.current_case.add_error_info(
message=self._exc_info_to_string(err, test))
super().addSubTest(test, subtest, err)
def __init__(self, board, riotdir, appdir, resultdir, junit=False):
self.board = board
self.riotdir = riotdir
self.appdir = appdir
self.resultdir = os.path.join(resultdir, appdir)
if junit:
if not junit_xml:
raise ImportError("`junit-xml` required for --report-xml")
self.testcase = junit_xml.TestCase(name=self.appdir,
stdout='', stderr='')
self.log_stream = io.StringIO()
logging.basicConfig(stream=self.log_stream)
else:
self.testcase = None
self.logger = logging.getLogger(f'{board}.{appdir}')
# Currently not handling absolute directories or outside of RIOT
assert is_in_directory(self.resultdir, resultdir), \
"Application result directory is outside main result directory"
def write_JUnit_XML(results, output_filename="output.xml"):
"""Write a JUnit XML test report to a file if junit_xml is available."""
try:
import junit_xml
except ImportError:
return
test_cases = []
for result in results:
test_name = reconstruct_test_name(result.command)
tc = junit_xml.TestCase(classname=test_name[0],
name=test_name[1],
elapsed_sec=result.wall_time,
stdout='\n'.join(result.stdout_lines),
stderr='\n'.join(result.stderr_lines))
if result.return_code == 0:
# Identify skipped tests
output = '\n'.join(result.stdout_lines + result.stderr_lines)
if re.search('skip', output, re.IGNORECASE):
# find first line including word 'skip' and use it as message
skipline = re.search('^((.*)skip(.*))$', output,
re.IGNORECASE | re.MULTILINE).group(1)
tc.add_skipped_info(skipline)
elif result.alert_status == Status.EXPECTED_FAIL:
tc.add_skipped_info("Expected test failure")
elif result.alert_status == Status.EXPECTED_UNSTABLE:
tc.add_skipped_info("Expected test instability")
else:
# Test failed. Extract error message and stack trace if possible
error_message = 'exit code %d' % result.return_code
error_output = '\n'.join(result.stderr_lines)
if result.stderr_lines:
error_message = result.stderr_lines[-1]
if len(result.stderr_lines) > 20:
error_output = '\n'.join(result.stderr_lines[-20:])
tc.add_failure_info(message=error_message, output=error_output)
test_cases.append(tc)
ts = junit_xml.TestSuite("libtbx.run_tests_parallel",
test_cases=test_cases)
with codecs.open(output_filename, "w", encoding="utf-8") as f:
ts.to_file(f, [ts], prettyprint=True, encoding="utf-8")
def wrapper(cfg):
global retcode
if cfg.get('junit'):
tstart = time.time()
tc = junit_xml.TestCase(func.__name__, classname="skt")
try:
func(cfg)
except Exception:
logging.error("Exception caught: %s", traceback.format_exc())
tc.add_failure_info(traceback.format_exc())
retcode = 1
# No exception but retcode != 0, probably tests failed
if retcode != 0 and not tc.is_failure():
tc.add_failure_info("Step finished with retcode: %d" % retcode)
tc.stdout = json.dumps(cfg, default=str)
tc.elapsed_sec = time.time() - tstart
cfg['_testcases'].append(tc)
else:
func(cfg)
def go(self):
""" Read executed tests and write junit """
super().go()
import_junit_xml()
suite = junit_xml.TestSuite(self.step.plan.name)
for result in self.step.plan.execute.results():
try:
main_log = self.step.plan.execute.read(result.log[0])
except (IndexError, AttributeError):
main_log = None
case = junit_xml.TestCase(result.name,
classname=None,
elapsed_sec=duration_to_seconds(
result.duration),
stdout=main_log)
# Map tmt OUTCOME to JUnit states
if result.result == "error":
case.add_error_info(result.result)
elif result.result == "fail":
case.add_failure_info(result.result)
def _end_test(self, test, type='Success', err_data=None, out_data=''):
test_index = self.testsRun - 1
# If already ended, do nothing
if (len(self.ran_tests) == self.testsRun and
self.ran_tests[test_index]):
return
start_time = self._time_tests[test_index]['start']
endtime = self._time_tests[test_index]['end'] or time.time()
if not self._time_tests[test_index]['end']:
self._time_tests[test_index]['end'] = endtime
test_name = str(test).split()[0]
test_classname = str(test).split()[-1][1:-1]
err_text = ''
if err_data:
err_text = self._exc_info_to_string(err_data, test)
testcase = junit_xml.TestCase(
name=test_name,
classname=test_classname,
elapsed_sec=(endtime - start_time),
stdout=out_data,
status=type
)
if testcase.status == 'Error':
testcase.add_error_info(
message='Error At {}'.format(test_name),
output=err_text
)
elif testcase.status == 'Failure':
testcase.add_failure_info(
message='Failure At {}'.format(test_name),
output=err_text
)
elif testcase.status == 'Skip':
testcase.add_skipped_info(
message='Skipped {}'.format(test_name),
output=out_data
)
self.ran_tests.append(testcase)
def wrapper(cfg):
"""
Outer wrapper of a @junit function.
Args:
cfg: A dictionary of skt configuration
"""
# pylint: disable=broad-except
global retcode
if cfg.get('junit'):
tstart = time.time()
testcase = junit_xml.TestCase(func.__name__, classname="skt")
try:
func(cfg)
except Exception:
logging.error(
"Unexpected exception caught, probably an "
"infrastructure failure or skt bug: %s",
traceback.format_exc())
testcase.add_error_info(traceback.format_exc())
retcode = SKT_ERROR
if retcode == SKT_FAIL:
# Tests failed
testcase.add_failure_info("Step finished with retcode: %d" %
retcode)
elif retcode >= SKT_ERROR:
testcase.add_error_info(
"Infrastructure issue or skt bug detected, retcode: %d" %
retcode)
testcase.stdout = json.dumps(cfg, default=str)
testcase.elapsed_sec = time.time() - tstart
cfg['_testcases'].append(testcase)
else:
func(cfg)
def results(request):
from pyresttest.resttest import RestTest
params = request.GET
test_groups = []
try:
test_file = params['file']
#lock.acquire()
try:
test_groups = RestTest().main({
"url": "",
"test": test_file,
"absolute_urls": True,
"log": "debug",
"interactive": False,
"ssl_insecure": True
}).items()
finally:
pass
#lock.release()
except OSError as e:
print e
traceback.print_exc
except IOError:
return HttpResponseServerError("Supplied test file was not found.")
except MultiValueDictKeyError:
return HttpResponseBadRequest(
"File was not supplied in the input params.")
# execute automated tests and validate responses and generate result json
passSteps = warnSteps = failSteps = passCount = warnCount = failCount = 0
import junit_xml as jx
try:
res = []
testSuites = []
for test_group in test_groups:
group_name = test_group[0]
tests = []
for test in test_group[1]:
if not test.step:
testCase = jx.TestCase(test.test.name, test.test._url, 0,
test.body)
if not test.passed:
testCase.add_failure_info(
output=str(test.failures[0].message))
tests.append(testCase)
if test.passed:
res.append({
"passed": "passed",
"name": test.test.name,
"group": group_name,
"response_code": test.response_code,
"step": test.step
})
if not test.step:
passCount += 1
else:
passSteps += 1
else:
if test.response_code == 204:
res.append({
"passed": "warning",
"name": test.test.name,
"group": group_name,
"response_code": test.response_code,
"failures": test.failures[0].message,
"step": test.step
})
if not test.step:
warnCount += 1
else:
warnSteps += 1
else:
res.append({
"passed": "failed",
"name": test.test.name,
"group": group_name,
"response_code": test.response_code,
"failures": test.failures[0].message,
"step": test.step
})
if not test.step:
failCount += 1
else:
failSteps += 1
testSuites.append(jx.TestSuite(group_name, tests))
res.append({
"pass": passCount,
"warn": warnCount,
"fail": failCount,
"passSteps": passSteps,
"warnSteps": warnSteps,
"failSteps": failSteps
})
except Exception as e:
print e
traceback.print_exc()
return HttpResponseServerError(
"Some unknown error occured, please verify that test file is correctly structured or not."
)
if params['format'] == 'json':
return HttpResponse(json.dumps(res)) # return results as json string.
elif params['format'] == 'xml':
return HttpResponse(jx.TestSuite.to_xml_string(testSuites))
def startTest(self, test):
self.current_case = junit.TestCase(name=str(test))
self.cases.append(self.current_case)
super().startTest(test)
def run_command(argv):
# pylint: disable=too-many-locals
this_dir = os.path.dirname(os.path.realpath(__file__))
top_dir = os.path.join(this_dir, "..", "..")
oparser = optparse.OptionParser(usage="usage: %prog [options]")
oparser.add_option("-f",
"--filter",
dest="filter",
metavar="REGEX",
type="string",
default=".*",
help="Run specific test according to a filter rule")
oparser.add_option("-l",
"--level",
dest="level",
type="string",
default="BASE",
help="Set test level threshold")
oparser.add_option("-i",
"--invert",
dest="invert_re",
action="store_true",
default=False,
help="Invert the matching of --filter")
oparser.add_option(
"-d",
"--dry-run",
dest="dry_run",
action="store_true",
default=False,
help="List tests that would have been executed and exit")
oparser.add_option("-p",
"--parallel",
dest="parallel",
type="string",
default="0,1",
help="Run every thest where test_ID % N == node_ID")
oparser.add_option("-o",
"--output",
dest="output",
type="string",
default=None,
help="Write XML output to file")
(opt, _) = oparser.parse_args(argv)
# Set test level threshold
if opt.level not in ElbeTestLevel.__members__:
print("Invalid level value '%s'. Valid values are: %s" %
(opt.level, ", ".join(key for key in ElbeTestLevel.__members__)))
os.sys.exit(20)
ElbeTestCase.level = ElbeTestLevel[opt.level]
# Find all tests
loader = unittest.defaultTestLoader
loader.suiteClass = ElbeTestSuite
suite = loader.discover(top_dir)
# then filter them
suite.filter_test(opt.parallel, opt.filter, opt.invert_re)
# Dry run? Just exit gently
if opt.dry_run:
suite.ls()
print(
"======================================================================\n"
"This was a dry run. No tests were executed")
os.sys.exit(0)
cases = []
err_cnt = 0
fail_cnt = 0
for test in suite:
print(test)
result = unittest.TestResult()
test.run(result)
case = junit.TestCase(name=str(test))
for error in result.errors:
case.add_error_info(message=error[1])
err_cnt += 1
for failure in result.failures:
case.add_failure_info(message=failure[1])
fail_cnt += 1
def run_tests(args):
"""
Run the user defined test scenario.
"""
TestImage.testRuns = args.testRepeats
TestImage.warmupRuns = args.testWarmups
# Create output location
if not os.path.exists("TestOutput"):
os.mkdir("TestOutput")
# Load test resources
binary = get_test_binary()
reference = get_test_reference_scores()
testList = get_test_listing(reference)
# Run tests
suites = []
suite = None
suiteFormat = None
# Run tests
maxCount = len(args.testBlockSize) * len(testList)
curCount = 0
statRun = 0
statSkip = 0
statPass = 0
for blockSize in args.testBlockSize:
for test in testList:
curCount += 1
# Skip tests not enabled for the current testing throughness level
if args.testLevel not in test.useLevel:
statSkip += 1
continue
# Skip tests not enabled for the current dynamic range level
if args.testRange not in test.useRange:
statSkip += 1
continue
# Skip tests not enabled for the current data format
if args.testFormat not in test.useFormat:
statSkip += 1
continue
# Start a new suite if the format changes
dat = (test.dynamicRange, test.format, blockSize)
testFormat = "%s.%s.%s" % dat
if (not suite) or (suiteFormat != testFormat):
suiteFormat = testFormat
suite = juxml.TestSuite("Image %s test suite" % suiteFormat)
suites.append(suite)
print("Running suite: %s" % suiteFormat)
# Run the test
test.run(binary, blockSize)
dat = (curCount, maxCount, test.name, blockSize,
test.runPSNR[blockSize], test.runTime[blockSize],
test.status[blockSize])
# Log results
statRun += 1
if "pass" in test.status[blockSize]:
statPass += 1
log = "Ran test %u/%u: %s %s, %0.3f dB, %0.3f s, %s" % dat
print(" + %s" % log)
# Generate JUnit result
caseName = "%s.%s" % (test.name, blockSize)
case = juxml.TestCase(caseName,
elapsed_sec=test.runTime[blockSize],
stdout=log)
suite.test_cases.append(case)
if test.status[blockSize] == "fail":
dat = (test.runPSNR[blockSize], test.referencePSNR[blockSize])
msg = "PSNR fail %0.3f dB is worse than %s dB" % dat
case.add_failure_info(msg)
# Print summary results
print("\nSummary")
if statRun == statPass:
print("+ PASS (%u ran)" % statRun)
else:
print("+ FAIL (%u ran, %u failed)" % (statRun, statRun - statPass))
# Write the JUnit results file
with open("TestOutput/results.xml", "w") as fileHandle:
juxml.TestSuite.to_file(fileHandle, suites)
if not sim_started:
sim_started = line.find('## run')==0
elif not sim_completed:
if 'Error:' in line:
sim_failed = True
if 'PASSED' in line:
sim_passed = True
# Check for exit, possibly might have tons of errors and clipped file
if 'Exiting xsim' in line:
sim_completed = True
except Exception as ex:
print('Failed parsing %s: %s'%(log,ex))
sim_error = True
# Translate results for JUnit.
test_name = os.path.basename(log).replace('simulate_','').replace('.log','')
case = junit_xml.TestCase(test_name, classname='pico_ethernet', timestamp=time.time())
if sim_error:
print('error')
case.add_error_info('Error: Failed parsing simulation results')
elif not sim_completed:
print('error')
case.add_error_info('Error: Simulation results appear to be truncated (could not find Exit message)')
elif sim_failed:
print('failed')
case.add_failure_info('Failure: Error detected in simulation')
elif sim_passed or IGNORE_MISSING_PASS_MSG:
print('passed')
pass # Success!
else:
case.add_error_info('Error: Invalid simulation log - unsure of result')
print('invalid')
def __call__(self):
path = self.node.et.text
test = junit.TestCase(name=path, classname=self.tag)
if not self.target.exists(path):
test.add_failure_info(message="FAILED")
return test
def make(args):
if args.skip_rules:
skip_rules = SkipRules(args.skip_rules)
params = get_params(args)
package = params.path.partition(os.path.sep)[0]
suite = junit_xml.TestSuite(
params.path,
timestamp=time.time(),
file=params.path,
package=package,
)
cmd = ['/usr/bin/env', 'make']
if params.directory:
cmd.extend(['-C', params.directory])
if params.makefile:
cmd.extend(['-f', params.makefile])
if args.jobs > 1:
cmd.extend(['-j', str(args.jobs)])
jobserver_auth_fds = get_jobserver_auth_fds()
targets = get_targets(args, params.path)
for target in targets:
if not target:
continue
cmd_target = cmd + [target]
timestamp = time.time()
start = time.monotonic()
# To better control the use of resources by a subprocess make, pass
# jobserver auth file descriptors in the subprocess, so it will
# participate in the jobserver protocol and schedule its jobs
# accordingly.
ret = subprocess.run(cmd_target,
pass_fds=jobserver_auth_fds,
capture_output=True,
text=True)
end = time.monotonic()
case_name = 'make ' + target
case_log = ' '.join(cmd_target)
case_classname = params.path
case = junit_xml.TestCase(
case_name,
classname=case_classname,
file=params.path,
log=case_log,
timestamp=timestamp,
elapsed_sec=(end - start),
stdout=ret.stdout,
stderr=ret.stderr,
)
suite.test_cases.append(case)
if ret.returncode != 0:
skipped, msg = skip_rules.match(params.path, ret.stderr)
if skipped:
case.add_skipped_info(msg)
continue
msg = 'Ended with non-zero exit code: {}'.format(ret.returncode)
case.add_failure_info(msg)
