def output_run_results_junit_xml(passing_tests: List[TestRunResult],
failing_tests: List[TestRunResult],
junit_dest: TextIO,
junit_merged_dest: TextIO):
'''Write results to JUnit XML
Two versions are produced: a normal version and a merged version. In the
normal version there is a test suite per unique test name with a different
test case per seed run. In the merged version there is a single test case
under the test suite with information for the individual runs merged
together. This is to aid use of the Azure Pipelines JUnit dashboard, which
doesn't neatly handle the test suite/test case hierarchy
'''
all_tests = passing_tests + failing_tests
test_suite_info = {}
for trr in all_tests:
# test_case_info contains a tuple per unique test name. The first
# element is a list of junit_xml.TestCase, one per test run with that
# name. The other merges together all of the test outputs to produce
# the merged output.
unmerged, merged = \
test_suite_info.setdefault(trr.name, ([], {'stdout': '',
'failures': ''}))
result_text = gen_test_run_result_text(trr)
# Create a test case for the TestRunResult. stdout holds the text
# describing the run. Add the same text to failures if the test failed.
test_case = junit_xml.TestCase(f'{trr.name}.{trr.seed}')
test_case.stdout = result_text
merged['stdout'] += result_text + '\n'
if not trr.passed:
test_case.add_failure_info(output=result_text)
merged['failures'] += result_text
unmerged.append(test_case)
# Output the normal JUnit XML
test_suites = [
junit_xml.TestSuite(name, test_cases)
for name, (test_cases, _) in test_suite_info.items()
]
junit_dest.write(junit_xml.to_xml_report_string(test_suites))
# Output the merged version of the JUnit XML
merged_test_suites = []
for name, (_, merged_test_info) in test_suite_info.items():
test_case = junit_xml.TestCase(name)
test_case.stdout = merged_test_info['stdout']
test_case.add_failure_info(output=merged_test_info['failures'])
merged_test_suites.append(junit_xml.TestSuite(name, [test_case]))
junit_merged_dest.write(junit_xml.to_xml_report_string(merged_test_suites))
def main():
"""This is the main entry point used by setup.cfg."""
# pylint: disable=protected-access
try:
global retcode
parser = setup_parser()
args = parser.parse_args()
check_args(parser, args)
setup_logging(args.verbose)
# We are gradually migrating away from messing with cfg and passing
# it everywhere.
if args.func in ['cmd_merge', 'cmd_build']:
args.func(args)
else:
cfg = load_config(args)
args.func(cfg)
if cfg.get('junit'):
testsuite = junit_xml.TestSuite("skt", cfg.get('_testcases'))
with open("%s/%s.xml" % (cfg.get('junit'), args._name),
'w') as fileh:
junit_xml.TestSuite.to_file(fileh, [testsuite])
sys.exit(retcode)
except KeyboardInterrupt:
print("\nExited at user request.")
sys.exit(130)
def format_baseline_for_junit_xml(baseline):
"""
:type baseline: dict
:rtype: str
"""
all_secrets = {}
for filename, secret_list in baseline['results'].items():
for secret in secret_list:
test_case = junit_xml.TestCase(
name="{}:{}".format(filename, secret["line_number"]))
test_case.add_failure_info(
message="Found secret of type {} on line {} in file {}".format(
secret["type"], secret["line_number"], filename),
failure_type=secret["type"])
if secret["type"] in all_secrets:
all_secrets[secret["type"]].append(test_case)
else:
all_secrets[secret["type"]] = [test_case]
test_suits = map(
lambda secret: junit_xml.TestSuite(name=secret[0],
test_cases=secret[1]),
all_secrets.items())
return junit_xml.to_xml_report_string(test_suits)
def main() -> None:
"""Generate a JUnit XML file."""
try:
log = logparser.logfile(sys.argv[1])
except IndexError:
sys.stderr.write(f"""\
\x1B[1mUsage: {sys.argv[0]} <logfile>\x1B[0m
""")
sys.exit(1)
except OSError:
sys.stderr.write(f"""\
\x1B[31mopen failed!\x1B[0m
\x1B[1mUsage: {sys.argv[0]} <logfile>\x1B[0m
""")
sys.exit(1)
toplevels = parse_log(log)
testcases: typing.List[junit_xml.TestCase] = []
for i, toplevel in enumerate(toplevels):
testcases += toplevel_to_junit(i, toplevel)
print(
json.dumps(list(map(lambda x: f"{x.classname}", testcases)), indent=4),
file=sys.stderr,
)
print(
junit_xml.TestSuite.to_xml_string(
[junit_xml.TestSuite("TBot", testcases)]))
def create_junit_output(tests):
""" Creates an output file for generating a junit test report
args:
tests: All tests completed + skipped
returns:
boolean: False if file generation failed - NOT YET
"""
# Populate junit objects for all performed tests
junit_tests = []
# Integration tests
for test in tests:
junit_object = jx.TestCase(test.name_, classname="IncludeOS.{}.{}".format(test.type_, test.category_))
# If test is skipped add skipped info
if test.skip_:
junit_object.add_skipped_info(message = test.skip_reason_, output = test.skip_reason_)
elif test.proc_.returncode is not 0:
junit_object.add_failure_info(output = test.output_[0])
else:
junit_object.stdout = test.output_[0]
junit_object.stderr = test.output_[1]
# Add to list of all test objects
junit_tests.append(junit_object)
# Stress and misc tests
ts = jx.TestSuite("IncludeOS tests", junit_tests)
with open('output.xml', 'w') as f:
jx.TestSuite.to_file(f, [ts], prettyprint=False)
def get_xml(self):
ts = junit.TestSuite(name="test", test_cases=self.cases)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
results = junit.TestSuite.to_xml_string([ts], encoding="utf-8")
return results
def summary(self, duration, example_count, failed_count, pending_count):
all_tests = []
for testgroup in self.junitxml_tests.keys():
for test in self.junitxml_tests[testgroup]['tests']:
all_tests.append(test)
return junit_xml.TestSuite(
name='{}.{}'.format('mamba', self.modulename),
test_cases=all_tests
)
def handle_composition(self, args: argparse.Namespace,
composition: mzcompose.Composition) -> None:
if args.workflow not in composition.workflows:
# Restart any dependencies whose definitions have changed. This is
# Docker Compose's default behavior for `up`, but not for `run`,
# which is a constant irritation that we paper over here. The trick,
# taken from Buildkite's Docker Compose plugin, is to run an `up`
# command that requests zero instances of the requested service.
if args.workflow:
composition.invoke(
"up",
"-d",
"--scale",
f"{args.workflow}=0",
args.workflow,
)
super().handle_composition(args, composition)
else:
# The user has specified a workflow rather than a service. Run the
# workflow instead of Docker Compose.
if args.unknown_args:
bad_arg = args.unknown_args[0]
elif args.unknown_subargs[0].startswith("-"):
bad_arg = args.unknown_subargs[0]
else:
bad_arg = None
if bad_arg:
raise UIError(
f"unknown option {bad_arg!r}",
hint=f"if {bad_arg!r} is a valid Docker Compose option, "
f"it can't be used when running {args.workflow!r}, because {args.workflow!r} "
"is a custom mzcompose workflow, not a Docker Compose service",
)
# Run the workflow inside of a test case so that we get some basic
# test analytics, even if the workflow doesn't define more granular
# test cases.
with composition.test_case(f"workflow-{args.workflow}"):
composition.workflow(args.workflow, *args.unknown_subargs[1:])
# Upload test report to Buildkite Test Analytics.
junit_suite = junit_xml.TestSuite(composition.name)
for (name, result) in composition.test_results.items():
test_case = junit_xml.TestCase(name, composition.name,
result.duration)
if result.error:
test_case.add_error_info(message=result.error)
junit_suite.test_cases.append(test_case)
junit_report = ci_util.junit_report_filename("mzcompose")
with junit_report.open("w") as f:
junit_xml.to_xml_report_file(f, [junit_suite])
ci_util.upload_junit_report("mzcompose", junit_report)
if any(result.error
for result in composition.test_results.values()):
raise UIError("at least one test case failed")
def __call__(self):
test_cases = []
for test in self.node:
try:
test_cases.append(self.do_test(test, self.target))
except TestException:
pass # TODO - Handle me!
ts = junit.TestSuite(name=self.node.et.attrib["name"],
test_cases=test_cases)
return ts
class JunitReport(object):
# wrapper for junit test report
# TODO: Don't support by multi-thread (although not likely to be used this way).
JUNIT_FILE_NAME = "XUNIT_RESULT.xml"
JUNIT_DEFAULT_TEST_SUITE = "test-suite"
JUNIT_TEST_SUITE = junit_xml.TestSuite(JUNIT_DEFAULT_TEST_SUITE)
JUNIT_CURRENT_TEST_CASE = None
_TEST_CASE_CREATED_TS = 0
@classmethod
def output_report(cls, junit_file_path):
""" Output current test result to file. """
with open(os.path.join(junit_file_path, cls.JUNIT_FILE_NAME),
"w") as f:
cls.JUNIT_TEST_SUITE.to_file(f, [cls.JUNIT_TEST_SUITE],
prettyprint=False)
@classmethod
def get_current_test_case(cls):
"""
By default, the test framework will handle junit test report automatically.
While some test case might want to update some info to test report.
They can use this method to get current test case created by test framework.
:return: current junit test case instance created by ``JunitTestReport.create_test_case``
"""
return cls.JUNIT_CURRENT_TEST_CASE
@classmethod
def test_case_finish(cls, test_case):
"""
Append the test case to test suite so it can be output to file.
Execution time will be automatically updated (compared to ``create_test_case``).
"""
test_case.elapsed_sec = time.time() - cls._TEST_CASE_CREATED_TS
cls.JUNIT_TEST_SUITE.test_cases.append(test_case)
@classmethod
def create_test_case(cls, name):
"""
Extend ``junit_xml.TestCase`` with:
1. save create test case so it can be get by ``get_current_test_case``
2. log create timestamp, so ``elapsed_sec`` can be auto updated in ``test_case_finish``.
:param name: test case name
:return: instance of ``junit_xml.TestCase``
"""
# set stdout to empty string, so we can always append string to stdout.
# It won't affect output logic. If stdout is empty, it won't be put to report.
test_case = junit_xml.TestCase(name, stdout="")
cls.JUNIT_CURRENT_TEST_CASE = test_case
cls._TEST_CASE_CREATED_TS = time.time()
return test_case
def emit_junit_xml(cluster_launch_attempts, frameworks):
"""Write out all the test actions failures to a junit file for jenkins or
similar"""
if not ok():
return
import junit_xml
launch_fake_testcases = []
for launch_attempt in cluster_launch_attempts:
attempt_duration = launch_attempt.end_time - launch_attempt.start_time
fake_test = junit_xml.TestCase(launch_attempt.name,
elapsed_sec=attempt_duration)
if launch_attempt.launch_succeeded:
fake_test.stdout = "Launch worked"
else:
fake_test.add_failure_info("Launch failed")
launch_fake_testcases.append(fake_test)
launch_suite = junit_xml.TestSuite("Cluster launches",
launch_fake_testcases)
fake_suites = []
fake_suites.append(launch_suite)
for framework in frameworks:
framework_testcases = []
for action_name, action in framework.actions.items():
action_duration = action['finish'] - action['start']
fake_test = junit_xml.TestCase(action_name,
elapsed_sec=action_duration,
stdout=action['stdout'],
stderr=action['stderr'])
if not action['ok']:
message = action['error_message']
if not message:
message = "%s failed" % action_name
fake_test.add_failure_info(message, action['error_output'])
framework_testcases.append(fake_test)
framework_suite = junit_xml.TestSuite("%s actions" % framework.name,
framework_testcases)
fake_suites.append(framework_suite)
with open("junit_testpy.xml", "w") as f:
junit_xml.TestSuite.to_file(f, fake_suites)
def create_junit_xml_file(projects, xml_file_path):
"""Create JUnit XML report file that can be used in Jenkins."""
synced_projects = []
for project in projects:
for k, v in project.items():
tc = junit_xml.TestCase(k)
tc.add_error_info(message=v[1])
synced_projects.append(tc)
ts = junit_xml.TestSuite("Sync", synced_projects)
with open(xml_file_path, 'w') as f:
junit_xml.TestSuite.to_file(f, [ts])
def main():
global retcode
parser = setup_parser()
args = parser.parse_args()
setup_logging(args.verbose)
cfg = load_config(args)
args.func(cfg)
if cfg.get('junit'):
ts = junit_xml.TestSuite("skt", cfg.get('_testcases'))
with open("%s/%s.xml" % (cfg.get('junit'), args._name), 'w') as fileh:
junit_xml.TestSuite.to_file(fileh, [ts])
sys.exit(retcode)
def write_JUnit_XML(results, output_filename="output.xml"):
"""Write a JUnit XML test report to a file if junit_xml is available."""
try:
import junit_xml
except ImportError:
return
def _decode_string(string):
try:
return string.encode('ascii', 'xmlcharrefreplace')
except Exception:
return unicode(string,
errors='ignore').encode('ascii',
'xmlcharrefreplace')
test_cases = []
for result in results:
test_name = reconstruct_test_name(result.command)
plain_stdout = map(_decode_string, result.stdout_lines)
plain_stderr = map(_decode_string, result.stderr_lines)
output = '\n'.join(plain_stdout + plain_stderr)
tc = junit_xml.TestCase(classname=test_name[0],
name=test_name[1],
elapsed_sec=result.wall_time,
stdout='\n'.join(plain_stdout),
stderr='\n'.join(plain_stderr))
if result.return_code == 0:
# Identify skipped tests
if re.search('skip', output, re.IGNORECASE):
# find first line including word 'skip' and use it as message
skipline = re.search('^((.*)skip(.*))$', output,
re.IGNORECASE | re.MULTILINE).group(1)
tc.add_skipped_info(skipline)
else:
# Test failed. Extract error message and stack trace if possible
error_message = 'exit code %d' % result.return_code
error_output = '\n'.join(plain_stderr)
if plain_stderr:
error_message = plain_stderr[-1]
if len(plain_stderr) > 20:
error_output = '\n'.join(plain_stderr[-20:])
tc.add_failure_info(message=error_message, output=error_output)
test_cases.append(tc)
ts = junit_xml.TestSuite("libtbx.run_tests_parallel",
test_cases=test_cases)
with open(output_filename, 'wb') as f:
ts.to_file(f, [ts], prettyprint=True)
def to_junit(
self,
suite_name="all_tests",
):
"""
Convert the tests to JUnit XML.
Returns a junit_xml.TestSuite containing all of the test cases. One test suite will be
generated with the name given in suite_name. Unity Fixture test groups are mapped to the
classname attribute of test cases; for basic Unity output there will be one class named
"default".
Optional arguments:
suite_name -- The name to use for the "name" and "package" attributes of the testsuite element.
Sample output:
<testsuite disabled="0" errors="0" failures="1" name="[suite_name]" package="[suite_name]" skipped="0" tests="8" time="0">
<testcase classname="test_group_1" name="group_1_test" />
<testcase classname="test_group_2" name="group_2_test" />
</testsuite>
"""
test_case_list = []
for test in self._tests:
if test.result() == "PASS":
test_case_list.append(
junit_xml.TestCase(name=test.name(),
classname=test.group()))
else:
junit_tc = junit_xml.TestCase(
name=test.name(),
classname=test.group(),
file=test.file(),
line=test.line(),
)
if test.result() == "FAIL":
junit_tc.add_failure_info(message=test.message(),
output=test.full_line())
elif test.result() == "IGNORE":
junit_tc.add_skipped_info(message=test.message(),
output=test.full_line())
test_case_list.append(junit_tc)
return junit_xml.TestSuite(name=suite_name,
package=suite_name,
test_cases=test_case_list)
def main() -> None:
"""Generate a JUnit XML file."""
log = logparser.from_argv()
toplevels = parse_log(log)
testcases: typing.List[junit_xml.TestCase] = []
for i, toplevel in enumerate(toplevels):
testcases += toplevel_to_junit(i, toplevel)
print(
json.dumps(list(map(lambda x: f"{x.classname}", testcases)), indent=4),
file=sys.stderr,
)
print(
junit_xml.TestSuite.to_xml_string(
[junit_xml.TestSuite("tbot", testcases)]))
def write_JUnit_XML(results, output_filename="output.xml"):
"""Write a JUnit XML test report to a file if junit_xml is available."""
try:
import junit_xml
except ImportError:
return
test_cases = []
for result in results:
test_name = reconstruct_test_name(result.command)
tc = junit_xml.TestCase(classname=test_name[0],
name=test_name[1],
elapsed_sec=result.wall_time,
stdout='\n'.join(result.stdout_lines),
stderr='\n'.join(result.stderr_lines))
if result.return_code == 0:
# Identify skipped tests
output = '\n'.join(result.stdout_lines + result.stderr_lines)
if re.search('skip', output, re.IGNORECASE):
# find first line including word 'skip' and use it as message
skipline = re.search('^((.*)skip(.*))$', output,
re.IGNORECASE | re.MULTILINE).group(1)
tc.add_skipped_info(skipline)
elif result.alert_status == Status.EXPECTED_FAIL:
tc.add_skipped_info("Expected test failure")
elif result.alert_status == Status.EXPECTED_UNSTABLE:
tc.add_skipped_info("Expected test instability")
else:
# Test failed. Extract error message and stack trace if possible
error_message = 'exit code %d' % result.return_code
error_output = '\n'.join(result.stderr_lines)
if result.stderr_lines:
error_message = result.stderr_lines[-1]
if len(result.stderr_lines) > 20:
error_output = '\n'.join(result.stderr_lines[-20:])
tc.add_failure_info(message=error_message, output=error_output)
test_cases.append(tc)
ts = junit_xml.TestSuite("libtbx.run_tests_parallel",
test_cases=test_cases)
with codecs.open(output_filename, "w", encoding="utf-8") as f:
ts.to_file(f, [ts], prettyprint=True, encoding="utf-8")
def _finalize_junit_domain(self, _tc, domain):
# Find all the junit-*.pickle files dropped by _mkreport()
# above and collect them into a testsuite, writing an XML in
# the CWD called junit.xml.
reports = []
domain_path_glob = os.path.join(tcfl.tc.tc_c.tmpdir, "junit", domain,
"*.pickle")
for filename in glob.glob(domain_path_glob):
with open(filename) as f:
jtc = cPickle.load(f)
reports.append(jtc)
ts = junit_xml.TestSuite(
self.junit_suite_name % _tc.kws,
reports,
# I'd like:
# hostname = _tc.kws['target_group_info'],
# but it can't, because each TC in the suite is run in a
# different target group. Maybe at some point TestCase
# will support hostname?
hostname=None,
id=_tc.kws['runid'],
package=self.junit_suite_package % _tc.kws,
timestamp=time.time(),
properties=self.junit_suite_properties, # Dictionary
)
del reports
if domain == "default":
junit_filename = "junit.xml"
else:
junit_filename = commonl.file_name_make_safe("junit-%s.xml" %
domain,
extra_chars="")
with codecs.open(junit_filename,
'w',
encoding='utf-8',
errors='ignore') as f:
junit_xml.TestSuite.to_file(f, [ts], prettyprint=True)
def go(self):
""" Read executed tests and write junit """
super().go()
import_junit_xml()
suite = junit_xml.TestSuite(self.step.plan.name)
for result in self.step.plan.execute.results():
try:
main_log = self.step.plan.execute.read(result.log[0])
except (IndexError, AttributeError):
main_log = None
case = junit_xml.TestCase(result.name,
classname=None,
elapsed_sec=duration_to_seconds(
result.duration),
stdout=main_log)
# Map tmt OUTCOME to JUnit states
if result.result == "error":
case.add_error_info(result.result)
elif result.result == "fail":
case.add_failure_info(result.result)
def run_fuzzer(fuzzer_name, fuzzer_dir, fuzzer_logdir, logger, will_retry):
def log(msg, *a, **k):
logger.log(msg, a, k, flush=True)
make_cmd = os.environ.get('MAKE', 'make')
make_flags = os.environ.get('MAKEFLAGS', '')
# Should run things?
if not should_run_submake(make_flags):
return 0
fuzzer_runok = os.path.join(fuzzer_dir, "run.ok")
if os.path.exists(fuzzer_runok):
last_modified = datetime.fromtimestamp(os.stat(fuzzer_runok).st_mtime)
log("Skipping as run.ok exists (updated @ {})",
last_modified.isoformat())
return 0
time_start = datetime.utcnow()
log("Starting @ {}", time_start.isoformat())
running_msg = "Running {} -C {} run (with MAKEFLAGS='{}')".format(
make_cmd,
fuzzer_dir,
make_flags,
)
log(running_msg)
log_suffix = ".{}.log".format(time_start.isoformat())
fuzzer_stdout = os.path.join(fuzzer_logdir, "stdout" + log_suffix)
fuzzer_stderr = os.path.join(fuzzer_logdir, "stderr" + log_suffix)
# Write header to stdout/stderr to make sure they match.
for fname in [fuzzer_stdout, fuzzer_stderr]:
with open(fname, "w") as fd:
fd.write("Build starting @ {}\n".format(time_start.isoformat()))
fd.write(running_msg)
fd.write("\n")
fd.write("-" * 75)
fd.write("\n")
fd.flush()
os.fsync(fd)
# Open the log files for appending
stdout_fd = open(fuzzer_stdout, "a")
stderr_fd = open(fuzzer_stderr, "a")
# Play nice with make's jobserver.
# See https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver # noqa
job_fds = []
if '--jobserver-fds' in make_flags:
job_re = re.search('--jobserver-fds=([0-9]+),([0-9]+)', make_flags)
assert job_re, make_flags
job_rd, job_wr = job_re.groups()
# Make copies of jobserver FDs in case a retry is needed.
job_rd = int(job_rd)
job_wr = int(job_wr)
assert job_rd > 2, (job_rd, job_wr, make_flags)
assert job_wr > 2, (job_rd, job_wr, make_flags)
job_fds.append(job_rd)
job_fds.append(job_wr)
p = None
try:
p = subprocess.Popen(
[make_cmd, '-C', fuzzer_dir, 'run'],
stdin=None,
stdout=stdout_fd,
stderr=stderr_fd,
pass_fds=job_fds,
)
while True:
try:
retcode = p.wait(timeout=10)
p = None
except subprocess.TimeoutExpired:
retcode = None
if retcode is not None:
break
mem = get_memory()['mem']
log(
"Still running (1m:{:0.2f}%, 5m:{:0.2f}%, 15m:{:0.2f}% Mem:{:0.1f}Gi used, {:0.1f}Gi free).\n{}",
*get_load(),
mem['used'] / 1e9,
mem['available'] /
1e9, # Using available so the numbers add up.
PsTree.get(p.pid),
)
except (Exception, KeyboardInterrupt, SystemExit):
retcode = -1
tb = io.StringIO()
traceback.print_exc(file=tb)
log(tb.getvalue())
# Prevent Ctrl-C so we exit properly...
old_sigint_handler = signal.getsignal(signal.SIGINT)
def log_die(sig, frame):
logger.log("Dieing!")
signal.signal(signal.SIGINT, log_die)
# Cleanup child process if they haven't already died.
try:
if p is not None:
try:
retcode = p.wait(1)
except subprocess.TimeoutExpired:
retcode = -1
p.kill()
p.wait()
log("Warning: Killed program which should have been dead!")
except Exception:
tb = io.StringIO()
traceback.print_exc(file=tb)
log(tb.getvalue())
# Wait for all children to finish.
try:
while True:
log("Child finished: {}", os.waitpid(-1, 0))
except ChildProcessError:
pass
log("Finishing ({}).", get_usage())
time_end = datetime.utcnow()
error_log = "\n".join(last_lines(open(fuzzer_stderr), 10000))
success_log = "\n".join(last_lines(open(fuzzer_stdout), 100))
# Find the next X_sponge_log.xml file name...
for i in range(0, 100):
tsfilename = os.path.join(fuzzer_logdir, '{}_sponge_log.xml'.format(i))
if not os.path.exists(tsfilename):
break
test_case = junit_xml.TestCase(
name=fuzzer_name,
timestamp=time_start.timestamp(),
elapsed_sec=(time_end - time_start).total_seconds(),
stdout=success_log,
stderr=error_log,
)
if retcode != 0:
test_case.add_failure_info(
'Fuzzer failed with exit code: {}'.format(retcode))
with open(tsfilename, 'w') as f:
ts = junit_xml.TestSuite(fuzzer_name, [test_case])
junit_xml.TestSuite.to_file(f, [ts])
if retcode != 0:
test_case.add_failure_info(
'Fuzzer failed with exit code: {}'.format(retcode), )
# Log the last 10,000 lines of stderr on a failure
log("""\
Failed @ {time_end} with exit code: {retcode}
--------------------------------------------------------------------------
{error_log}
--------------------------------------------------------------------------
Failed @ {time_end} with exit code: {retcode}
""",
retcode=retcode,
error_log=error_log,
time_end=time_end.isoformat())
else:
# Log the last 100 lines of a successful run
log(
"""\
Succeeded! @ {}
--------------------------------------------------------------------------
{}
--------------------------------------------------------------------------
Succeeded! @ {}
""", time_end.isoformat(), success_log, time_end.isoformat())
logger.flush()
signal.signal(signal.SIGINT, old_sigint_handler)
return retcode
def main(args):
"""For one board, compile all examples and tests and run test on board."""
logger = logging.getLogger(args.board)
if args.loglevel:
loglevel = logging.getLevelName(args.loglevel.upper())
logger.setLevel(loglevel)
logger.addHandler(LOG_HANDLER)
logger.info('Saving toolchain')
save_toolchain(args.riot_directory, args.result_directory)
board = check_is_board(args.riot_directory, args.board)
logger.debug('board: %s', board)
# Expand application directories: allows use of glob in application names
apps_dirs = _expand_apps_directories(args.applications,
args.riot_directory)
apps_dirs_skip = _expand_apps_directories(args.applications_exclude,
args.riot_directory, skip=True)
app_dirs = apps_directories(args.riot_directory, apps_dirs=apps_dirs,
apps_dirs_skip=apps_dirs_skip)
logger.debug('app_dirs: %s', app_dirs)
logger.debug('resultdir: %s', args.result_directory)
board_result_directory = os.path.join(args.result_directory, args.board)
# Overwrite the compile/test targets from command line arguments
RIOTApplication.COMPILE_TARGETS = args.compile_targets
RIOTApplication.FLASH_TARGETS = args.flash_targets
RIOTApplication.TEST_TARGETS = args.test_targets
RIOTApplication.TEST_AVAILABLE_TARGETS = args.test_available_targets
# List of applications for board
applications = [RIOTApplication(board, args.riot_directory, app_dir,
board_result_directory,
junit=args.report_xml)
for app_dir in app_dirs]
# Execute tests
errors = [app.run_compilation_and_test(clean_after=args.clean_after,
runtest=not args.no_test,
incremental=args.incremental,
jobs=args.jobs,
with_test_only=args.with_test_only)
for app in applications]
errors = [e for e in errors if e is not None]
num_errors = len(errors)
summary = _test_failed_summary(errors, relpathstart=board_result_directory)
save_failure_summary(board_result_directory, summary)
if args.report_xml:
if not junit_xml:
raise ImportError("`junit-xml` required for --report-xml")
report_file = os.path.join(board_result_directory, "report.xml")
with open(report_file, "w+", encoding="utf-8") as report:
junit_xml.TestSuite.to_file(
report,
[junit_xml.TestSuite(f'compile_and_test_for_{board}',
[app.testcase for app in applications])]
)
if num_errors:
logger.error('Tests failed: %d', num_errors)
print(summary, end='')
else:
logger.info('Tests successful')
sys.exit(num_errors)
def main(): # type: () -> int
args = arg_parser().parse_args(sys.argv[1:])
if '--' in args.args:
args.args.remove('--')
# Remove test arguments with wrong syntax
if args.testargs is not None:
args.testargs = [
testarg for testarg in args.testargs if testarg.count('==') == 1
]
if not args.test:
arg_parser().print_help()
return 1
with open(args.test) as f:
tests = yaml.load(f, Loader=yaml.SafeLoader)
failures = 0
unsupported = 0
passed = 0
suite_name, _ = os.path.splitext(os.path.basename(args.test))
report = junit_xml.TestSuite(suite_name, [])
if args.only_tools:
alltests = tests
tests = []
for t in alltests:
loader = schema_salad.ref_resolver.Loader({"id": "@id"})
cwl = loader.resolve_ref(t["tool"])[0]
if isinstance(cwl, dict):
if cwl["class"] == "CommandLineTool":
tests.append(t)
else:
raise Exception("Unexpected code path.")
if args.l:
for i, t in enumerate(tests):
if t.get("short_name"):
print(u"[%i] %s: %s" %
(i + 1, t["short_name"], t["doc"].strip()))
else:
print(u"[%i] %s" % (i + 1, t["doc"].strip()))
return 0
if args.n is not None or args.s is not None:
ntest = []
if args.n is not None:
for s in args.n.split(","):
sp = s.split("-")
if len(sp) == 2:
ntest.extend(list(range(int(sp[0]) - 1, int(sp[1]))))
else:
ntest.append(int(s) - 1)
if args.s is not None:
for s in args.s.split(","):
test_number = get_test_number_by_key(tests, "short_name", s)
if test_number:
ntest.append(test_number)
else:
_logger.error('Test with short name "%s" not found ', s)
return 1
else:
ntest = list(range(0, len(tests)))
total = 0
with ThreadPoolExecutor(max_workers=args.j) as executor:
jobs = [
executor.submit(run_test, args, tests[i], i + 1, len(tests),
args.timeout) for i in ntest
]
try:
for i, job in zip(ntest, jobs):
test_result = job.result()
test_case = test_result.create_test_case(tests[i])
total += 1
return_code = test_result.return_code
category = test_case.category
if return_code == 0:
passed += 1
elif return_code != 0 and return_code != UNSUPPORTED_FEATURE:
failures += 1
test_case.add_failure_info(output=test_result.message)
elif return_code == UNSUPPORTED_FEATURE and category == REQUIRED:
failures += 1
test_case.add_failure_info(output=test_result.message)
elif category != REQUIRED and return_code == UNSUPPORTED_FEATURE:
unsupported += 1
test_case.add_skipped_info("Unsupported")
else:
raise Exception(
"This is impossible, return_code: {}, category: "
"{}".format(return_code, category))
report.test_cases.append(test_case)
except KeyboardInterrupt:
for job in jobs:
job.cancel()
_logger.error("Tests interrupted")
if args.junit_xml:
with open(args.junit_xml, 'w') as xml:
junit_xml.TestSuite.to_file(xml, [report])
if failures == 0 and unsupported == 0:
_logger.info("All tests passed")
return 0
if failures == 0 and unsupported > 0:
_logger.warning("%i tests passed, %i unsupported features",
total - unsupported, unsupported)
return 0
_logger.warning("%i tests passed, %i failures, %i unsupported features",
total - (failures + unsupported), failures, unsupported)
return 1
for error in result.errors:
case.add_error_info(message=error[1])
err_cnt += 1
for failure in result.failures:
case.add_failure_info(message=failure[1])
fail_cnt += 1
for us in result.unexpectedSuccesses:
case.add_failure_info(message=us)
err_cnt += 1
for skip in result.skipped:
case.add_skipped_info(message=skip[1])
cases.append(case)
ts = junit.TestSuite(name="test", test_cases=cases)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
results = junit.TestSuite.to_xml_string([ts], encoding="utf-8")
if opt.output is None:
print(results)
else:
with open(opt.output, "w") as f:
f.write(results)
os.sys.exit(err_cnt | fail_cnt)
def main(): # type: () -> int
args = arg_parser().parse_args(sys.argv[1:])
if "--" in args.args:
args.args.remove("--")
# Remove test arguments with wrong syntax
if args.testargs is not None:
args.testargs = [
testarg for testarg in args.testargs if testarg.count("==") == 1
]
if not args.test:
arg_parser().print_help()
return 1
schema_resource = pkg_resources.resource_stream(__name__,
"cwltest-schema.yml")
cache = {
"https://w3id.org/cwl/cwltest/cwltest-schema.yml":
schema_resource.read().decode("utf-8")
} # type: Optional[Dict[str, Union[str, Graph, bool]]]
(document_loader, avsc_names, schema_metadata,
metaschema_loader) = schema_salad.schema.load_schema(
"https://w3id.org/cwl/cwltest/cwltest-schema.yml", cache=cache)
if not isinstance(avsc_names, schema_salad.avro.schema.Names):
print(avsc_names)
return 1
tests, metadata = schema_salad.schema.load_and_validate(
document_loader, avsc_names, args.test, True)
failures = 0
unsupported = 0
passed = 0
suite_name, _ = os.path.splitext(os.path.basename(args.test))
report = junit_xml.TestSuite(suite_name, [])
# the number of total tests, failured tests, unsupported tests and passed tests for each tag
ntotal = defaultdict(int) # type: Dict[str, int]
nfailures = defaultdict(int) # type: Dict[str, int]
nunsupported = defaultdict(int) # type: Dict[str, int]
npassed = defaultdict(int) # type: Dict[str, int]
if args.only_tools:
alltests = tests
tests = []
for t in alltests:
loader = schema_salad.ref_resolver.Loader({"id": "@id"})
cwl = loader.resolve_ref(t["tool"])[0]
if isinstance(cwl, dict):
if cwl["class"] == "CommandLineTool":
tests.append(t)
else:
raise Exception("Unexpected code path.")
if args.tags:
alltests = tests
tests = []
tags = args.tags.split(",")
for t in alltests:
ts = t.get("tags", [])
if any((tag in ts for tag in tags)):
tests.append(t)
for t in tests:
if t.get("label"):
t["short_name"] = t["label"]
if args.show_tags:
alltags = set() # type: Set[str]
for t in tests:
ts = t.get("tags", [])
alltags |= set(ts)
for tag in alltags:
print(tag)
return 0
if args.l:
for i, t in enumerate(tests):
if t.get("short_name"):
print(u"[%i] %s: %s" %
(i + 1, t["short_name"], t.get("doc", "").strip()))
else:
print(u"[%i] %s" % (i + 1, t.get("doc", "").strip()))
return 0
if args.n is not None or args.s is not None:
ntest = []
if args.n is not None:
for s in args.n.split(","):
sp = s.split("-")
if len(sp) == 2:
ntest.extend(list(range(int(sp[0]) - 1, int(sp[1]))))
else:
ntest.append(int(s) - 1)
if args.s is not None:
for s in args.s.split(","):
test_number = get_test_number_by_key(tests, "short_name", s)
if test_number:
ntest.append(test_number)
else:
_logger.error('Test with short name "%s" not found ', s)
return 1
else:
ntest = list(range(0, len(tests)))
total = 0
with ThreadPoolExecutor(max_workers=args.j) as executor:
jobs = [
executor.submit(
run_test,
args,
tests[i],
i + 1,
len(tests),
args.timeout,
args.junit_verbose,
) for i in ntest
]
try:
for i, job in zip(ntest, jobs):
test_result = job.result()
test_case = test_result.create_test_case(tests[i])
test_case.url = "cwltest:{}#{}".format(suite_name, i + 1)
total += 1
tags = tests[i].get("tags", [])
for t in tags:
ntotal[t] += 1
return_code = test_result.return_code
category = test_case.category
if return_code == 0:
passed += 1
for t in tags:
npassed[t] += 1
elif return_code != 0 and return_code != UNSUPPORTED_FEATURE:
failures += 1
for t in tags:
nfailures[t] += 1
test_case.add_failure_info(output=test_result.message)
elif return_code == UNSUPPORTED_FEATURE and category == REQUIRED:
failures += 1
for t in tags:
nfailures[t] += 1
test_case.add_failure_info(output=test_result.message)
elif category != REQUIRED and return_code == UNSUPPORTED_FEATURE:
unsupported += 1
for t in tags:
nunsupported[t] += 1
test_case.add_skipped_info("Unsupported")
else:
raise Exception(
"This is impossible, return_code: {}, category: "
"{}".format(return_code, category))
report.test_cases.append(test_case)
except KeyboardInterrupt:
for job in jobs:
job.cancel()
_logger.error("Tests interrupted")
if args.junit_xml:
with open(args.junit_xml, "w") as xml:
junit_xml.TestSuite.to_file(xml, [report])
if args.badgedir:
os.mkdir(args.badgedir)
for t, v in ntotal.items():
percent = int((npassed[t] / float(v)) * 100)
if npassed[t] == v:
color = "green"
elif t == "required":
color = "red"
else:
color = "yellow"
with open("{}/{}.json".format(args.badgedir, t), "w") as out:
out.write(
json.dumps({
"subject": "{}".format(t),
"status": "{}%".format(percent),
"color": color,
}))
if failures == 0 and unsupported == 0:
_logger.info("All tests passed")
return 0
if failures == 0 and unsupported > 0:
_logger.warning("%i tests passed, %i unsupported features",
total - unsupported, unsupported)
return 0
_logger.warning(
"%i tests passed, %i failures, %i unsupported features",
total - (failures + unsupported),
failures,
unsupported,
)
return 1
def main(): # type: () -> int
parser = argparse.ArgumentParser(
description='Compliance tests for cwltool')
parser.add_argument("--test",
type=str,
help="YAML file describing test cases",
required=True)
parser.add_argument("--basedir",
type=str,
help="Basedir to use for tests",
default=".")
parser.add_argument("-l", action="store_true", help="List tests then exit")
parser.add_argument("-n",
type=str,
default=None,
help="Run a specific tests, format is 1,3-6,9")
parser.add_argument(
"--tool",
type=str,
default="cwl-runner",
help="CWL runner executable to use (default 'cwl-runner'")
parser.add_argument("--only-tools",
action="store_true",
help="Only test CommandLineTools")
parser.add_argument("--junit-xml",
type=str,
default=None,
help="Path to JUnit xml file")
parser.add_argument("--test-arg",
type=str,
help="Additional argument given in test cases and "
"required prefix for tool runner.",
metavar="cache==--cache-dir",
action="append",
dest="testargs")
parser.add_argument("args",
help="arguments to pass first to tool runner",
nargs=argparse.REMAINDER)
parser.add_argument(
"-j",
type=int,
default=1,
help="Specifies the number of tests to run simultaneously "
"(defaults to one).")
parser.add_argument("--verbose",
action="store_true",
help="More verbose output during test run.")
args = parser.parse_args()
if '--' in args.args:
args.args.remove('--')
# Remove test arguments with wrong syntax
args.testargs = [
testarg for testarg in args.testargs if testarg.count('==') == 1
]
if not args.test:
parser.print_help()
return 1
with open(args.test) as f:
tests = yaml.load(f, Loader=yaml.SafeLoader)
failures = 0
unsupported = 0
passed = 0
suite_name, _ = os.path.splitext(os.path.basename(args.test))
report = junit_xml.TestSuite(suite_name, [])
if args.only_tools:
alltests = tests
tests = []
for t in alltests:
loader = schema_salad.ref_resolver.Loader({"id": "@id"})
cwl = loader.resolve_ref(t["tool"])[0]
if isinstance(cwl, dict):
if cwl["class"] == "CommandLineTool":
tests.append(t)
else:
raise Exception("Unexpected code path.")
if args.l:
for i, t in enumerate(tests):
print u"[%i] %s" % (i + 1, t["doc"].strip())
return 0
if args.n is not None:
ntest = []
for s in args.n.split(","):
sp = s.split("-")
if len(sp) == 2:
ntest.extend(range(int(sp[0]) - 1, int(sp[1])))
else:
ntest.append(int(s) - 1)
else:
ntest = range(0, len(tests))
total = 0
with ThreadPoolExecutor(max_workers=args.j) as executor:
jobs = [executor.submit(run_test, args, i, tests) for i in ntest]
try:
for i, job in zip(ntest, jobs):
test_result = job.result()
test_case = test_result.create_test_case(tests[i])
total += 1
if test_result.return_code == 1:
failures += 1
test_case.add_failure_info(output=test_result.message)
elif test_result.return_code == UNSUPPORTED_FEATURE:
unsupported += 1
test_case.add_skipped_info("Unsupported")
else:
passed += 1
report.test_cases.append(test_case)
except KeyboardInterrupt:
for job in jobs:
job.cancel()
_logger.error("Tests interrupted")
if args.junit_xml:
with open(args.junit_xml, 'w') as fp:
junit_xml.TestSuite.to_file(fp, [report])
if failures == 0 and unsupported == 0:
_logger.info("All tests passed")
return 0
elif failures == 0 and unsupported > 0:
_logger.warn("%i tests passed, %i unsupported features",
total - unsupported, unsupported)
return 0
else:
_logger.warn("%i tests passed, %i failures, %i unsupported features",
total - (failures + unsupported), failures, unsupported)
return 1
classname=case.suite.name,
timestamp=case.logged,
)
if case.result == TestCase.RESULT_FAIL:
logs = None
# TODO: is this of any use? (yaml inside xml!)
if (case.start_log_line is not None
and case.end_log_line is not None):
logs = logs_instance.read(self.get_object(),
case.start_log_line,
case.end_log_line)
tc.add_error_info("failed", output=logs)
elif case.result == TestCase.RESULT_SKIP:
tc.add_skipped_info("skipped")
cases.append(tc)
suites.append(junit_xml.TestSuite(suite.name, test_cases=cases))
data = junit_xml.TestSuite.to_xml_string(suites, encoding="utf-8")
response = HttpResponse(data, content_type="application/xml")
response["Content-Disposition"] = ("attachment; filename=job_%d.xml" %
self.get_object().id)
return response
@detail_route(methods=["get"], suffix="logs")
def logs(self, request, **kwargs):
start = safe_str2int(request.query_params.get("start", 0))
end = safe_str2int(request.query_params.get("end", None))
try:
if start == 0 and end is None:
data = logs_instance.open(self.get_object())
response = FileResponse(data, content_type="application/yaml")
def run_tests(args):
"""
Run the user defined test scenario.
"""
TestImage.testRuns = args.testRepeats
TestImage.warmupRuns = args.testWarmups
# Create output location
if not os.path.exists("TestOutput"):
os.mkdir("TestOutput")
# Load test resources
binary = get_test_binary()
reference = get_test_reference_scores()
testList = get_test_listing(reference)
# Run tests
suites = []
suite = None
suiteFormat = None
# Run tests
maxCount = len(args.testBlockSize) * len(testList)
curCount = 0
statRun = 0
statSkip = 0
statPass = 0
for blockSize in args.testBlockSize:
for test in testList:
curCount += 1
# Skip tests not enabled for the current testing throughness level
if args.testLevel not in test.useLevel:
statSkip += 1
continue
# Skip tests not enabled for the current dynamic range level
if args.testRange not in test.useRange:
statSkip += 1
continue
# Skip tests not enabled for the current data format
if args.testFormat not in test.useFormat:
statSkip += 1
continue
# Start a new suite if the format changes
dat = (test.dynamicRange, test.format, blockSize)
testFormat = "%s.%s.%s" % dat
if (not suite) or (suiteFormat != testFormat):
suiteFormat = testFormat
suite = juxml.TestSuite("Image %s test suite" % suiteFormat)
suites.append(suite)
print("Running suite: %s" % suiteFormat)
# Run the test
test.run(binary, blockSize)
dat = (curCount, maxCount, test.name, blockSize,
test.runPSNR[blockSize], test.runTime[blockSize],
test.status[blockSize])
# Log results
statRun += 1
if "pass" in test.status[blockSize]:
statPass += 1
log = "Ran test %u/%u: %s %s, %0.3f dB, %0.3f s, %s" % dat
print(" + %s" % log)
# Generate JUnit result
caseName = "%s.%s" % (test.name, blockSize)
case = juxml.TestCase(caseName,
elapsed_sec=test.runTime[blockSize],
stdout=log)
suite.test_cases.append(case)
if test.status[blockSize] == "fail":
dat = (test.runPSNR[blockSize], test.referencePSNR[blockSize])
msg = "PSNR fail %0.3f dB is worse than %s dB" % dat
case.add_failure_info(msg)
# Print summary results
print("\nSummary")
if statRun == statPass:
print("+ PASS (%u ran)" % statRun)
else:
print("+ FAIL (%u ran, %u failed)" % (statRun, statRun - statPass))
# Write the JUnit results file
with open("TestOutput/results.xml", "w") as fileHandle:
juxml.TestSuite.to_file(fileHandle, suites)
sim_passed = True
# Check for exit, possibly might have tons of errors and clipped file
if 'Exiting xsim' in line:
sim_completed = True
except Exception as ex:
print('Failed parsing %s: %s'%(log,ex))
sim_error = True
# Translate results for JUnit.
test_name = os.path.basename(log).replace('simulate_','').replace('.log','')
case = junit_xml.TestCase(test_name, classname='pico_ethernet', timestamp=time.time())
if sim_error:
print('error')
case.add_error_info('Error: Failed parsing simulation results')
elif not sim_completed:
print('error')
case.add_error_info('Error: Simulation results appear to be truncated (could not find Exit message)')
elif sim_failed:
print('failed')
case.add_failure_info('Failure: Error detected in simulation')
elif sim_passed or IGNORE_MISSING_PASS_MSG:
print('passed')
pass # Success!
else:
case.add_error_info('Error: Invalid simulation log - unsure of result')
print('invalid')
test_cases.append(case)
test_suite = junit_xml.TestSuite('pico_ethernet', test_cases)
with open('sim_results.xml','wt') as fh:
junit_xml.TestSuite.to_file(fh, [test_suite])
def get_test_suite(self, module_name):
return junit_xml.TestSuite(
name=module_name,
test_cases=self.ran_tests
)
class JunitReport(object):
# wrapper for junit test report
# TODO: JunitReport methods are not thread safe (although not likely to be used this way).
JUNIT_FILE_NAME = 'XUNIT_RESULT.xml'
JUNIT_DEFAULT_TEST_SUITE = 'test-suite'
JUNIT_TEST_SUITE = junit_xml.TestSuite(JUNIT_DEFAULT_TEST_SUITE,
hostname=socket.gethostname(),
timestamp=datetime.utcnow().isoformat())
JUNIT_CURRENT_TEST_CASE = None
_TEST_CASE_CREATED_TS = 0
@classmethod
def output_report(cls, junit_file_path):
""" Output current test result to file. """
with open(os.path.join(junit_file_path, cls.JUNIT_FILE_NAME), 'w') as f:
junit_xml.to_xml_report_file(f, [cls.JUNIT_TEST_SUITE], prettyprint=False)
@classmethod
def get_current_test_case(cls):
"""
By default, the test framework will handle junit test report automatically.
While some test case might want to update some info to test report.
They can use this method to get current test case created by test framework.
:return: current junit test case instance created by ``JunitTestReport.create_test_case``
"""
return cls.JUNIT_CURRENT_TEST_CASE
@classmethod
def test_case_finish(cls, test_case):
"""
Append the test case to test suite so it can be output to file.
Execution time will be automatically updated (compared to ``create_test_case``).
"""
test_case.elapsed_sec = time.time() - cls._TEST_CASE_CREATED_TS
cls.JUNIT_TEST_SUITE.test_cases.append(test_case)
@classmethod
def create_test_case(cls, name):
"""
Extend ``junit_xml.TestCase`` with:
1. save create test case so it can be get by ``get_current_test_case``
2. log create timestamp, so ``elapsed_sec`` can be auto updated in ``test_case_finish``.
:param name: test case name
:return: instance of ``junit_xml.TestCase``
"""
# set stdout to empty string, so we can always append string to stdout.
# It won't affect output logic. If stdout is empty, it won't be put to report.
test_case = junit_xml.TestCase(name, stdout='')
cls.JUNIT_CURRENT_TEST_CASE = test_case
cls._TEST_CASE_CREATED_TS = time.time()
return test_case
@classmethod
def update_performance(cls, performance_items):
"""
Update performance results to ``stdout`` of current test case.
:param performance_items: a list of performance items. each performance item is a key-value pair.
"""
assert cls.JUNIT_CURRENT_TEST_CASE
for item in performance_items:
cls.JUNIT_CURRENT_TEST_CASE.stdout += '[{}]: {}\n'.format(item[0], item[1])
