def test_parse_failed_targets_error_raise(self):
registry = TestRegistry({})
with temporary_dir() as junit_xml_dir:
junit_xml_file = os.path.join(junit_xml_dir, 'TEST-bad.xml')
with open(junit_xml_file, 'w') as fp:
fp.write('<invalid></xml>')
with self.assertRaises(ParseError) as exc:
parse_failed_targets(registry, junit_xml_dir, self._raise_handler)
self.assertEqual(junit_xml_file, exc.exception.junit_xml_path)
self.assertIsInstance(exc.exception.cause, XmlParser.XmlError)
def test_parse_failed_targets_error_continue(self):
registry = RegistryOfTests({})
with temporary_dir() as junit_xml_dir:
bad_file1 = os.path.join(junit_xml_dir, 'TEST-bad1.xml')
with open(bad_file1, 'w') as fp:
fp.write('<testsuite failures="nan" errors="0"/>')
with open(os.path.join(junit_xml_dir, 'TEST-good.xml'), 'w') as fp:
fp.write("""
<testsuite failures="0" errors="1">
<testcase classname="org.pantsbuild.Error" name="testError">
<error/>
</testcase>
</testsuite>
""")
bad_file2 = os.path.join(junit_xml_dir, 'TEST-bad2.xml')
with open(bad_file2, 'w') as fp:
fp.write('<invalid></xml>')
collect_handler = self.CollectHandler()
failed_targets = parse_failed_targets(registry, junit_xml_dir,
collect_handler)
self.assertEqual(2, len(collect_handler.errors))
self.assertEqual({bad_file1, bad_file2},
{e.xml_path
for e in collect_handler.errors})
self.assertEqual(
{None: {JUnitTest('org.pantsbuild.Error', 'testError')}},
failed_targets)
def test_parse_failed_targets_nominal(self):
registry = RegistryOfTests({
JUnitTest('org.pantsbuild.Failure'):
'Bob',
JUnitTest('org.pantsbuild.Error'):
'Jane',
JUnitTest('org.pantsbuild.AnotherError'):
'Bob',
JUnitTest('org.pantsbuild.subpackage.AnotherFailure'):
'Mary'
})
with temporary_dir() as junit_xml_dir:
with open(os.path.join(junit_xml_dir, 'TEST-a.xml'), 'w') as fp:
fp.write("""
<testsuite failures="1" errors="1">
<testcase classname="org.pantsbuild.Green" name="testOK"/>
<testcase classname="org.pantsbuild.Failure" name="testFailure">
<failure/>
</testcase>
<testcase classname="org.pantsbuild.Error" name="testError">
<error/>
</testcase>
</testsuite>
""")
with open(os.path.join(junit_xml_dir, 'TEST-b.xml'), 'w') as fp:
fp.write("""
<testsuite failures="0" errors="1">
<testcase classname="org.pantsbuild.AnotherError" name="testAnotherError">
<error/>
</testcase>
</testsuite>
""")
with open(os.path.join(junit_xml_dir, 'random.xml'), 'w') as fp:
fp.write('<invalid></xml>')
with safe_open(os.path.join(junit_xml_dir, 'subdir', 'TEST-c.xml'),
'w') as fp:
fp.write("""
<testsuite failures="1" errors="0">
<testcase classname="org.pantsbuild.subpackage.AnotherFailure" name="testAnotherFailue">
<failure/>
</testcase>
</testsuite>
""")
failed_targets = parse_failed_targets(registry, junit_xml_dir,
self._raise_handler)
self.assertEqual(
{
'Bob': {
JUnitTest('org.pantsbuild.Failure', 'testFailure'),
JUnitTest('org.pantsbuild.AnotherError',
'testAnotherError')
},
'Jane': {JUnitTest('org.pantsbuild.Error', 'testError')},
'Mary': {
JUnitTest('org.pantsbuild.subpackage.AnotherFailure',
'testAnotherFailue')
}
}, failed_targets)
def test_parse_failed_targets_no_files(self):
registry = RegistryOfTests({})
with temporary_dir() as junit_xml_dir:
failed_targets = parse_failed_targets(registry, junit_xml_dir,
self._raise_handler)
self.assertEqual({}, failed_targets)
def test_parse_failed_targets_nominal(self):
registry = RegistryOfTests({JUnitTest('org.pantsbuild.Failure'): 'Bob',
JUnitTest('org.pantsbuild.Error'): 'Jane',
JUnitTest('org.pantsbuild.AnotherError'): 'Bob',
JUnitTest('org.pantsbuild.subpackage.AnotherFailure'): 'Mary'})
with temporary_dir() as junit_xml_dir:
with open(os.path.join(junit_xml_dir, 'TEST-a.xml'), 'w') as fp:
fp.write("""
<testsuite failures="1" errors="1">
<testcase classname="org.pantsbuild.Green" name="testOK"/>
<testcase classname="org.pantsbuild.Failure" name="testFailure">
<failure/>
</testcase>
<testcase classname="org.pantsbuild.Error" name="testError">
<error/>
</testcase>
</testsuite>
""")
with open(os.path.join(junit_xml_dir, 'TEST-b.xml'), 'w') as fp:
fp.write("""
<testsuite failures="0" errors="1">
<testcase classname="org.pantsbuild.AnotherError" name="testAnotherError">
<error/>
</testcase>
</testsuite>
""")
with open(os.path.join(junit_xml_dir, 'random.xml'), 'w') as fp:
fp.write('<invalid></xml>')
with safe_open(os.path.join(junit_xml_dir, 'subdir', 'TEST-c.xml'), 'w') as fp:
fp.write("""
<testsuite failures="1" errors="0">
<testcase classname="org.pantsbuild.subpackage.AnotherFailure" name="testAnotherFailue">
<failure/>
</testcase>
</testsuite>
""")
failed_targets = parse_failed_targets(registry, junit_xml_dir, self._raise_handler)
self.assertEqual({'Bob': {JUnitTest('org.pantsbuild.Failure', 'testFailure'),
JUnitTest('org.pantsbuild.AnotherError', 'testAnotherError')},
'Jane': {JUnitTest('org.pantsbuild.Error', 'testError')},
'Mary': {JUnitTest('org.pantsbuild.subpackage.AnotherFailure',
'testAnotherFailue')}},
failed_targets)
def _get_failed_targets_from_junitxml(self, junitxml, targets):
# Note that unlike in Java, we can't easily map targets to test classnames up-front.
# Instead we grab the classnames seen in practice and work backwards to the targets.
# First map the dotted paths of the modules to their respective targets.
pex_src_root = os.path.relpath(
self.context.products.get_data(
GatherSources.PYTHON_SOURCES).path(), get_buildroot())
dotted_module_path_to_target = {}
for target in targets:
for src in target.sources_relative_to_source_root():
pex_src = os.path.join(pex_src_root, src)
dotted_path = os.path.splitext(pex_src)[0].replace(
os.path.sep, '.')
dotted_module_path_to_target[dotted_path] = target
# Now grab the classnames from the xml file.
xml = XmlParser.from_file(junitxml)
classname_and_names = (
(testcase.getAttribute('classname'), testcase.getAttribute('name'))
for testcase in xml.parsed.getElementsByTagName('testcase'))
# Now find which module each classname belongs to, and map it to its target.
test_target_pairs = []
for classname, name in classname_and_names:
# if the classname is empty, it means that there was an error in the module body,
# outside any class or method body. In this case the module name in its entirety
# ends up in the 'name' attribute.
dotted_path = classname or name
while dotted_path and dotted_path not in dotted_module_path_to_target:
dotted_path = dotted_path.rpartition('.')[0]
if dotted_path:
target = dotted_module_path_to_target[dotted_path]
test_target_pairs.append((Test(classname), target))
# Now parse the junit xml the usual way.
def error_handler(e):
raise TaskError(e)
failed_targets_map = parse_failed_targets(
TestRegistry(test_target_pairs), junitxml, error_handler)
return failed_targets_map.keys()
def test_parse_failed_targets_error_continue(self):
registry = TestRegistry({})
with temporary_dir() as junit_xml_dir:
bad_file1 = os.path.join(junit_xml_dir, 'TEST-bad1.xml')
with open(bad_file1, 'w') as fp:
fp.write('<testsuite failures="nan" errors="0"/>')
with open(os.path.join(junit_xml_dir, 'TEST-good.xml'), 'w') as fp:
fp.write("""
<testsuite failures="0" errors="1">
<testcase classname="org.pantsbuild.Error" name="testError">
<error/>
</testcase>
</testsuite>
""")
bad_file2 = os.path.join(junit_xml_dir, 'TEST-bad2.xml')
with open(bad_file2, 'w') as fp:
fp.write('<invalid></xml>')
collect_handler = self.CollectHandler()
failed_targets = parse_failed_targets(registry, junit_xml_dir, collect_handler)
self.assertEqual(2, len(collect_handler.errors))
self.assertEqual({bad_file1, bad_file2}, {e.junit_xml_path for e in collect_handler.errors})
self.assertEqual({None: {JUnitTest('org.pantsbuild.Error', 'testError')}}, failed_targets)
def run_tests(self, fail_fast, test_targets, output_dir, coverage):
test_registry = self._collect_test_targets(test_targets)
if test_registry.empty:
return TestResult.rc(0)
coverage.instrument(output_dir)
def parse_error_handler(parse_error):
# Just log and move on since the result is only used to characterize failures, and raising
# an error here would just distract from the underlying test failures.
self.context.log.error(
'Error parsing test result file {path}: {cause}'.format(
path=parse_error.xml_path, cause=parse_error.cause))
# The 'instrument_classpath' product below below will be `None` if not set, and we'll default
# back to runtime_classpath
classpath_product = self.context.products.get_data(
'instrument_classpath')
result = 0
for batch_id, (properties,
batch) in enumerate(self._iter_batches(test_registry)):
(workdir, platform, target_jvm_options, target_env_vars,
concurrency, threads) = properties
batch_output_dir = output_dir
if self._batched:
batch_output_dir = os.path.join(batch_output_dir,
'batch-{}'.format(batch_id))
run_modifications = coverage.run_modifications(batch_output_dir)
extra_jvm_options = run_modifications.extra_jvm_options
# Batches of test classes will likely exist within the same targets: dedupe them.
relevant_targets = {
test_registry.get_owning_target(t)
for t in batch
}
complete_classpath = OrderedSet()
complete_classpath.update(run_modifications.classpath_prepend)
complete_classpath.update(JUnit.global_instance().runner_classpath(
self.context))
complete_classpath.update(
self.classpath(relevant_targets,
classpath_product=classpath_product))
distribution = JvmPlatform.preferred_jvm_distribution(
[platform], self._strict_jvm_version)
# Override cmdline args with values from junit_test() target that specify concurrency:
args = self._args(fail_fast, batch_output_dir) + [u'-xmlreport']
if concurrency is not None:
args = remove_arg(args, '-default-parallel')
if concurrency == JUnitTests.CONCURRENCY_SERIAL:
args = ensure_arg(args,
'-default-concurrency',
param='SERIAL')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES:
args = ensure_arg(args,
'-default-concurrency',
param='PARALLEL_CLASSES')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS:
args = ensure_arg(args,
'-default-concurrency',
param='PARALLEL_METHODS')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS:
args = ensure_arg(args,
'-default-concurrency',
param='PARALLEL_CLASSES_AND_METHODS')
if threads is not None:
args = remove_arg(args, '-parallel-threads', has_param=True)
args += ['-parallel-threads', str(threads)]
batch_test_specs = [test.render_test_spec() for test in batch]
with argfile.safe_args(batch_test_specs,
self.get_options()) as batch_tests:
with self._chroot(relevant_targets, workdir) as chroot:
self.context.log.debug('CWD = {}'.format(chroot))
self.context.log.debug('platform = {}'.format(platform))
with environment_as(**dict(target_env_vars)):
subprocess_result = self._spawn_and_wait(
executor=SubprocessExecutor(distribution),
distribution=distribution,
classpath=complete_classpath,
main=JUnit.RUNNER_MAIN,
jvm_options=self.jvm_options + extra_jvm_options +
list(target_jvm_options),
args=args + batch_tests,
workunit_factory=self.context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnitLabel.TEST],
cwd=chroot,
synthetic_jar_dir=batch_output_dir,
create_synthetic_jar=self.synthetic_classpath,
)
self.context.log.debug(
'JUnit subprocess exited with result ({})'.format(
subprocess_result))
result += abs(subprocess_result)
tests_info = self.parse_test_info(batch_output_dir,
parse_error_handler,
['classname'])
for test_name, test_info in tests_info.items():
test_item = Test(test_info['classname'], test_name)
test_target = test_registry.get_owning_target(test_item)
self.report_all_info_for_single_test(
self.options_scope, test_target, test_name, test_info)
if result != 0 and fail_fast:
break
if result == 0:
return TestResult.rc(0)
target_to_failed_test = parse_failed_targets(test_registry, output_dir,
parse_error_handler)
def sort_owning_target(t):
return t.address.spec if t else None
failed_targets = sorted(target_to_failed_test, key=sort_owning_target)
error_message_lines = []
if self._failure_summary:
def render_owning_target(t):
return t.address.reference() if t else '<Unknown Target>'
for target in failed_targets:
error_message_lines.append('\n{indent}{owner}'.format(
indent=' ' * 4, owner=render_owning_target(target)))
for test in sorted(target_to_failed_test[target]):
error_message_lines.append(
'{indent}{classname}#{methodname}'.format(
indent=' ' * 8,
classname=test.classname,
methodname=test.methodname))
error_message_lines.append(
'\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
.format(main=JUnit.RUNNER_MAIN,
code=result,
failed=len(failed_targets),
targets=pluralize(len(failed_targets), 'target')))
return TestResult(msg='\n'.join(error_message_lines),
rc=result,
failed_targets=failed_targets)
def run_tests(self, fail_fast, test_targets, output_dir, coverage):
test_registry = self._collect_test_targets(test_targets)
if test_registry.empty:
return TestResult.rc(0)
coverage.instrument(output_dir)
def parse_error_handler(parse_error):
# Just log and move on since the result is only used to characterize failures, and raising
# an error here would just distract from the underlying test failures.
self.context.log.error('Error parsing test result file {path}: {cause}'
.format(path=parse_error.xml_path, cause=parse_error.cause))
# The 'instrument_classpath' product below below will be `None` if not set, and we'll default
# back to runtime_classpath
classpath_product = self.context.products.get_data('instrument_classpath')
result = 0
for batch_id, (properties, batch) in enumerate(self._iter_batches(test_registry)):
(workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties
batch_output_dir = output_dir
if self._batched:
batch_output_dir = os.path.join(batch_output_dir, 'batch-{}'.format(batch_id))
run_modifications = coverage.run_modifications(batch_output_dir)
extra_jvm_options = run_modifications.extra_jvm_options
# Batches of test classes will likely exist within the same targets: dedupe them.
relevant_targets = {test_registry.get_owning_target(t) for t in batch}
complete_classpath = OrderedSet()
complete_classpath.update(run_modifications.classpath_prepend)
complete_classpath.update(JUnit.global_instance().runner_classpath(self.context))
complete_classpath.update(self.classpath(relevant_targets,
classpath_product=classpath_product))
distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version)
# Override cmdline args with values from junit_test() target that specify concurrency:
args = self._args(fail_fast, batch_output_dir) + [u'-xmlreport']
if concurrency is not None:
args = remove_arg(args, '-default-parallel')
if concurrency == JUnitTests.CONCURRENCY_SERIAL:
args = ensure_arg(args, '-default-concurrency', param='SERIAL')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES:
args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS:
args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS:
args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS')
if threads is not None:
args = remove_arg(args, '-parallel-threads', has_param=True)
args += ['-parallel-threads', str(threads)]
batch_test_specs = [test.render_test_spec() for test in batch]
with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests:
with self._chroot(relevant_targets, workdir) as chroot:
self.context.log.debug('CWD = {}'.format(chroot))
self.context.log.debug('platform = {}'.format(platform))
with environment_as(**dict(target_env_vars)):
subprocess_result = self._spawn_and_wait(
executor=SubprocessExecutor(distribution),
distribution=distribution,
classpath=complete_classpath,
main=JUnit.RUNNER_MAIN,
jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options),
args=args + batch_tests,
workunit_factory=self.context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnitLabel.TEST],
cwd=chroot,
synthetic_jar_dir=batch_output_dir,
create_synthetic_jar=self.synthetic_classpath,
)
self.context.log.debug('JUnit subprocess exited with result ({})'
.format(subprocess_result))
result += abs(subprocess_result)
tests_info = self.parse_test_info(batch_output_dir, parse_error_handler, ['classname'])
for test_name, test_info in tests_info.items():
test_item = Test(test_info['classname'], test_name)
test_target = test_registry.get_owning_target(test_item)
self.report_all_info_for_single_test(self.options_scope, test_target,
test_name, test_info)
if result != 0 and fail_fast:
break
if result == 0:
return TestResult.rc(0)
target_to_failed_test = parse_failed_targets(test_registry, output_dir, parse_error_handler)
def sort_owning_target(t):
return t.address.spec if t else None
failed_targets = sorted(target_to_failed_test, key=sort_owning_target)
error_message_lines = []
if self._failure_summary:
def render_owning_target(t):
return t.address.reference() if t else '<Unknown Target>'
for target in failed_targets:
error_message_lines.append('\n{indent}{owner}'.format(indent=' ' * 4,
owner=render_owning_target(target)))
for test in sorted(target_to_failed_test[target]):
error_message_lines.append('{indent}{classname}#{methodname}'
.format(indent=' ' * 8,
classname=test.classname,
methodname=test.methodname))
error_message_lines.append(
'\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
.format(main=JUnit.RUNNER_MAIN, code=result, failed=len(failed_targets),
targets=pluralize(len(failed_targets), 'target'))
)
return TestResult(msg='\n'.join(error_message_lines), rc=result, failed_targets=failed_targets)
def _run_tests(self, test_registry, output_dir, coverage=None):
if coverage:
extra_jvm_options = coverage.extra_jvm_options
classpath_prepend = coverage.classpath_prepend
classpath_append = coverage.classpath_append
else:
extra_jvm_options = []
classpath_prepend = ()
classpath_append = ()
tests_by_properties = test_registry.index(
lambda tgt: tgt.cwd if tgt.cwd is not None else self._working_dir,
lambda tgt: tgt.test_platform,
lambda tgt: tgt.payload.extra_jvm_options,
lambda tgt: tgt.payload.extra_env_vars,
lambda tgt: tgt.concurrency, lambda tgt: tgt.threads)
# the below will be None if not set, and we'll default back to runtime_classpath
classpath_product = self.context.products.get_data(
'instrument_classpath')
result = 0
for properties, tests in tests_by_properties.items():
(workdir, platform, target_jvm_options, target_env_vars,
concurrency, threads) = properties
for batch in self._partition(tests):
# Batches of test classes will likely exist within the same targets: dedupe them.
relevant_targets = {
test_registry.get_owning_target(t)
for t in batch
}
complete_classpath = OrderedSet()
complete_classpath.update(classpath_prepend)
complete_classpath.update(
JUnit.global_instance().runner_classpath(self.context))
complete_classpath.update(
self.classpath(relevant_targets,
classpath_product=classpath_product))
complete_classpath.update(classpath_append)
distribution = JvmPlatform.preferred_jvm_distribution(
[platform], self._strict_jvm_version)
# Override cmdline args with values from junit_test() target that specify concurrency:
args = self._args(output_dir) + [u'-xmlreport']
if concurrency is not None:
args = remove_arg(args, '-default-parallel')
if concurrency == JUnitTests.CONCURRENCY_SERIAL:
args = ensure_arg(args,
'-default-concurrency',
param='SERIAL')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES:
args = ensure_arg(args,
'-default-concurrency',
param='PARALLEL_CLASSES')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS:
args = ensure_arg(args,
'-default-concurrency',
param='PARALLEL_METHODS')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS:
args = ensure_arg(args,
'-default-concurrency',
param='PARALLEL_CLASSES_AND_METHODS')
if threads is not None:
args = remove_arg(args,
'-parallel-threads',
has_param=True)
args += ['-parallel-threads', str(threads)]
batch_test_specs = [test.render_test_spec() for test in batch]
with argfile.safe_args(batch_test_specs,
self.get_options()) as batch_tests:
self.context.log.debug('CWD = {}'.format(workdir))
self.context.log.debug('platform = {}'.format(platform))
with environment_as(**dict(target_env_vars)):
result += abs(
self._spawn_and_wait(
executor=SubprocessExecutor(distribution),
distribution=distribution,
classpath=complete_classpath,
main=JUnit.RUNNER_MAIN,
jvm_options=self.jvm_options +
extra_jvm_options + list(target_jvm_options),
args=args + batch_tests,
workunit_factory=self.context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnitLabel.TEST],
cwd=workdir,
synthetic_jar_dir=output_dir,
create_synthetic_jar=self.synthetic_classpath,
))
if result != 0 and self._fail_fast:
break
if result != 0:
def error_handler(parse_error):
# Just log and move on since the result is only used to characterize failures, and raising
# an error here would just distract from the underlying test failures.
self.context.log.error(
'Error parsing test result file {path}: {cause}'.format(
path=parse_error.junit_xml_path,
cause=parse_error.cause))
target_to_failed_test = parse_failed_targets(
test_registry, output_dir, error_handler)
failed_targets = sorted(target_to_failed_test,
key=lambda t: t.address.spec)
error_message_lines = []
if self._failure_summary:
for target in failed_targets:
error_message_lines.append('\n{indent}{address}'.format(
indent=' ' * 4, address=target.address.spec))
for test in sorted(target_to_failed_test[target]):
error_message_lines.append(
'{indent}{classname}#{methodname}'.format(
indent=' ' * 8,
classname=test.classname,
methodname=test.methodname))
error_message_lines.append(
'\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
.format(main=JUnit.RUNNER_MAIN,
code=result,
failed=len(failed_targets),
targets=pluralize(len(failed_targets), 'target')))
raise ErrorWhileTesting('\n'.join(error_message_lines),
failed_targets=list(failed_targets))
def run_tests(self, fail_fast, test_targets, output_dir, coverage, complete_test_registry):
test_registry = complete_test_registry.filter(test_targets)
if test_registry.empty:
return TestResult.successful
coverage.instrument(output_dir)
def parse_error_handler(parse_error):
# Just log and move on since the result is only used to characterize failures, and raising
# an error here would just distract from the underlying test failures.
self.context.log.error(
"Error parsing test result file {path}: {cause}".format(
path=parse_error.xml_path, cause=parse_error.cause
)
)
# The 'instrument_classpath' product below below will be `None` if not set, and we'll default
# back to runtime_classpath
classpath_product = self.context.products.get_data("instrument_classpath")
result = 0
for batch_id, (properties, batch) in enumerate(self._iter_batches(test_registry)):
(
workdir,
platform,
target_jvm_options,
target_env_vars,
concurrency,
threads,
) = properties
batch_output_dir = output_dir
if self._batched:
batch_output_dir = os.path.join(batch_output_dir, f"batch-{batch_id}")
run_modifications = coverage.run_modifications(batch_output_dir)
self.context.log.debug(f"run_modifications: {run_modifications}")
extra_jvm_options = run_modifications.extra_jvm_options
# Batches of test classes will likely exist within the same targets: dedupe them.
relevant_targets = {test_registry.get_owning_target(t) for t in batch}
complete_classpath = OrderedSet()
complete_classpath.update(run_modifications.classpath_prepend)
complete_classpath.update(JUnit.global_instance().runner_classpath(self.context))
complete_classpath.update(
self.classpath(relevant_targets, classpath_product=classpath_product)
)
distribution = self.preferred_jvm_distribution([platform], self._strict_jvm_version)
# Override cmdline args with values from junit_test() target that specify concurrency:
args = self._args(fail_fast, batch_output_dir) + ["-xmlreport"]
if concurrency is not None:
args = remove_arg(args, "-default-parallel")
if concurrency == JUnitTests.CONCURRENCY_SERIAL:
args = ensure_arg(args, "-default-concurrency", param="SERIAL")
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES:
args = ensure_arg(args, "-default-concurrency", param="PARALLEL_CLASSES")
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS:
args = ensure_arg(args, "-default-concurrency", param="PARALLEL_METHODS")
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS:
args = ensure_arg(
args, "-default-concurrency", param="PARALLEL_CLASSES_AND_METHODS"
)
if threads is not None:
args = remove_arg(args, "-parallel-threads", has_param=True)
args += ["-parallel-threads", str(threads)]
batch_test_specs = [test.render_test_spec() for test in batch]
with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests:
with self.chroot(relevant_targets, workdir) as chroot:
self.context.log.debug(f"CWD = {chroot}")
self.context.log.debug(f"platform = {platform}")
with environment_as(**dict(target_env_vars)):
subprocess_result = self.spawn_and_wait(
relevant_targets,
executor=SubprocessExecutor(distribution),
distribution=distribution,
classpath=complete_classpath,
main=JUnit.RUNNER_MAIN,
jvm_options=self.jvm_options
+ list(platform.jvm_options)
+ extra_jvm_options
+ list(target_jvm_options),
args=args + batch_tests,
workunit_factory=self.context.new_workunit,
workunit_name="run",
workunit_labels=[WorkUnitLabel.TEST],
cwd=chroot,
synthetic_jar_dir=batch_output_dir,
create_synthetic_jar=self.synthetic_classpath,
)
self.context.log.debug(
"JUnit subprocess exited with result ({})".format(subprocess_result)
)
result += abs(subprocess_result)
tests_info = self.parse_test_info(
batch_output_dir, parse_error_handler, ["classname"]
)
for test_name, test_info in tests_info.items():
test_item = Test(test_info["classname"], test_name)
test_target = test_registry.get_owning_target(test_item)
self.report_all_info_for_single_test(
self.options_scope, test_target, test_name, test_info
)
if result != 0 and fail_fast:
break
if result == 0:
return TestResult.successful
# NB: If the TestRegistry fails to find the owning target of a failed test, the target key in
# this dictionary will be None: helper methods in this block account for that.
target_to_failed_test = parse_failed_targets(test_registry, output_dir, parse_error_handler)
def sort_owning_target(t):
return t.address.spec if t else ""
failed_targets = sorted(target_to_failed_test, key=sort_owning_target)
error_message_lines = []
if self._failure_summary:
def render_owning_target(t):
return t.address.reference() if t else "<Unknown Target>"
for target in failed_targets:
error_message_lines.append(f"\n{(' ' * 4)}{render_owning_target(target)}")
for test in sorted(target_to_failed_test[target]):
error_message_lines.append(f"{' ' * 8}{test.classname}#{test.methodname}")
error_message_lines.append(
"\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.".format(
main=JUnit.RUNNER_MAIN,
code=result,
failed=len(failed_targets),
targets=pluralize(len(failed_targets), "target"),
)
)
return TestResult(
msg="\n".join(error_message_lines), rc=result, failed_targets=failed_targets
)
def test_parse_failed_targets_no_files(self):
registry = TestRegistry({})
with temporary_dir() as junit_xml_dir:
failed_targets = parse_failed_targets(registry, junit_xml_dir, self._raise_handler)
self.assertEqual({}, failed_targets)
def _run_tests(self, test_registry, output_dir, coverage=None):
if coverage:
extra_jvm_options = coverage.extra_jvm_options
classpath_prepend = coverage.classpath_prepend
classpath_append = coverage.classpath_append
else:
extra_jvm_options = []
classpath_prepend = ()
classpath_append = ()
tests_by_properties = test_registry.index(
lambda tgt: tgt.cwd if tgt.cwd is not None else self._working_dir,
lambda tgt: tgt.test_platform,
lambda tgt: tgt.payload.extra_jvm_options,
lambda tgt: tgt.payload.extra_env_vars,
lambda tgt: tgt.concurrency,
lambda tgt: tgt.threads)
# the below will be None if not set, and we'll default back to runtime_classpath
classpath_product = self.context.products.get_data('instrument_classpath')
result = 0
for properties, tests in tests_by_properties.items():
(workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties
for batch in self._partition(tests):
# Batches of test classes will likely exist within the same targets: dedupe them.
relevant_targets = {test_registry.get_owning_target(t) for t in batch}
complete_classpath = OrderedSet()
complete_classpath.update(classpath_prepend)
complete_classpath.update(JUnit.global_instance().runner_classpath(self.context))
complete_classpath.update(self.classpath(relevant_targets,
classpath_product=classpath_product))
complete_classpath.update(classpath_append)
distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version)
# Override cmdline args with values from junit_test() target that specify concurrency:
args = self._args(output_dir) + [u'-xmlreport']
if concurrency is not None:
args = remove_arg(args, '-default-parallel')
if concurrency == JUnitTests.CONCURRENCY_SERIAL:
args = ensure_arg(args, '-default-concurrency', param='SERIAL')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES:
args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS:
args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS')
elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS:
args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS')
if threads is not None:
args = remove_arg(args, '-parallel-threads', has_param=True)
args += ['-parallel-threads', str(threads)]
batch_test_specs = [test.render_test_spec() for test in batch]
with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests:
self.context.log.debug('CWD = {}'.format(workdir))
self.context.log.debug('platform = {}'.format(platform))
with environment_as(**dict(target_env_vars)):
result += abs(self._spawn_and_wait(
executor=SubprocessExecutor(distribution),
distribution=distribution,
classpath=complete_classpath,
main=JUnit.RUNNER_MAIN,
jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options),
args=args + batch_tests,
workunit_factory=self.context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnitLabel.TEST],
cwd=workdir,
synthetic_jar_dir=output_dir,
create_synthetic_jar=self.synthetic_classpath,
))
if result != 0 and self._fail_fast:
break
if result != 0:
def error_handler(parse_error):
# Just log and move on since the result is only used to characterize failures, and raising
# an error here would just distract from the underlying test failures.
self.context.log.error('Error parsing test result file {path}: {cause}'
.format(path=parse_error.junit_xml_path, cause=parse_error.cause))
target_to_failed_test = parse_failed_targets(test_registry, output_dir, error_handler)
failed_targets = sorted(target_to_failed_test, key=lambda t: t.address.spec)
error_message_lines = []
if self._failure_summary:
for target in failed_targets:
error_message_lines.append('\n{indent}{address}'.format(indent=' ' * 4,
address=target.address.spec))
for test in sorted(target_to_failed_test[target]):
error_message_lines.append('{indent}{classname}#{methodname}'
.format(indent=' ' * 8,
classname=test.classname,
methodname=test.methodname))
error_message_lines.append(
'\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
.format(main=JUnit.RUNNER_MAIN, code=result, failed=len(failed_targets),
targets=pluralize(len(failed_targets), 'target'))
)
raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
def test_parse_failed_targets_nominal(self):
registry = RegistryOfTests({
JUnitTest("org.pantsbuild.Failure"):
"Bob",
JUnitTest("org.pantsbuild.Error"):
"Jane",
JUnitTest("org.pantsbuild.AnotherError"):
"Bob",
JUnitTest("org.pantsbuild.subpackage.AnotherFailure"):
"Mary",
})
with temporary_dir() as junit_xml_dir:
with open(os.path.join(junit_xml_dir, "TEST-a.xml"), "w") as fp:
fp.write("""
<testsuite failures="1" errors="1">
<testcase classname="org.pantsbuild.Green" name="testOK"/>
<testcase classname="org.pantsbuild.Failure" name="testFailure">
<failure/>
</testcase>
<testcase classname="org.pantsbuild.Error" name="testError">
<error/>
</testcase>
</testsuite>
""")
with open(os.path.join(junit_xml_dir, "TEST-b.xml"), "w") as fp:
fp.write("""
<testsuite failures="0" errors="1">
<testcase classname="org.pantsbuild.AnotherError" name="testAnotherError">
<error/>
</testcase>
</testsuite>
""")
with open(os.path.join(junit_xml_dir, "random.xml"), "w") as fp:
fp.write("<invalid></xml>")
with safe_open(os.path.join(junit_xml_dir, "subdir", "TEST-c.xml"),
"w") as fp:
fp.write("""
<testsuite failures="1" errors="0">
<testcase classname="org.pantsbuild.subpackage.AnotherFailure" name="testAnotherFailure">
<failure/>
</testcase>
</testsuite>
""")
failed_targets = parse_failed_targets(registry, junit_xml_dir,
self._raise_handler)
self.assertEqual(
{
"Bob": {
JUnitTest("org.pantsbuild.Failure", "testFailure"),
JUnitTest("org.pantsbuild.AnotherError",
"testAnotherError"),
},
"Jane": {JUnitTest("org.pantsbuild.Error", "testError")},
"Mary": {
JUnitTest("org.pantsbuild.subpackage.AnotherFailure",
"testAnotherFailure")
},
},
failed_targets,
)
