def visualize_execution_graph(scheduler, request):
with temporary_file_path(cleanup=False, suffix='.dot') as dot_file:
scheduler.visualize_graph_to_file(request.roots, dot_file)
print('dot file saved to: {}'.format(dot_file))
with temporary_file_path(cleanup=False, suffix='.svg') as image_file:
subprocess.check_call('dot -Tsvg -o{} {}'.format(image_file, dot_file), shell=True)
print('svg file saved to: {}'.format(image_file))
binary_util.ui_open(image_file)
def _do_run_tests(self, targets, workunit):
def _extract_resultlog_filename(args):
resultlogs = [
arg[arg.find('=') + 1:] for arg in args
if arg.startswith('--resultlog=')
]
if resultlogs:
return resultlogs[0]
else:
try:
return args[args.index('--resultlog') + 1]
except IndexError:
self.context.log.error(
'--resultlog specified without an argument')
return None
except ValueError:
return None
if not targets:
return PythonTestResult.rc(0)
sources = list(
itertools.chain(
*[t.sources_relative_to_buildroot() for t in targets]))
if not sources:
return PythonTestResult.rc(0)
with self._test_runner(targets, workunit) as (pex, test_args):
def run_and_analyze(resultlog_path):
result = self._do_run_tests_with_args(pex, workunit, args)
failed_targets = self._get_failed_targets_from_resultlogs(
resultlog_path, targets)
return result.with_failed_targets(failed_targets)
# N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the
# top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs)
# from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726
args = ['--confcutdir', get_buildroot()]
if self.get_options().fail_fast:
args.extend(['-x'])
if self._debug:
args.extend(['-s'])
if self.get_options().colors:
args.extend(['--color', 'yes'])
for options in self.get_options().options + self.get_passthru_args(
):
args.extend(safe_shlex_split(options))
args.extend(test_args)
args.extend(sources)
# The user might have already specified the resultlog option. In such case, reuse it.
resultlog_arg = _extract_resultlog_filename(args)
if resultlog_arg:
return run_and_analyze(resultlog_arg)
else:
with temporary_file_path() as resultlog_path:
args.insert(0, '--resultlog={0}'.format(resultlog_path))
return run_and_analyze(resultlog_path)
def _do_run_tests(self, targets, workunit):
if not targets:
return PythonTestResult.rc(0)
sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets]))
if not sources:
return PythonTestResult.rc(0)
with self._test_runner(targets, workunit) as (pex, test_args):
def run_and_analyze(resultlog_path):
result = self._do_run_tests_with_args(pex, workunit, args)
failed_targets = self._get_failed_targets_from_resultlogs(resultlog_path, targets)
return result.with_failed_targets(failed_targets)
args = []
if self._debug:
args.extend(['-s'])
if self.get_options().colors:
args.extend(['--color', 'yes'])
for options in self.get_options().options + self.get_passthru_args():
args.extend(safe_shlex_split(options))
args.extend(test_args)
args.extend(sources)
# The user might have already specified the resultlog option. In such case, reuse it.
resultlogs = [arg.split('=', 1)[-1] for arg in args if arg.startswith('--resultlog=')]
if resultlogs:
return run_and_analyze(resultlogs[-1])
else:
with temporary_file_path() as resultlog_path:
args.append('--resultlog={0}'.format(resultlog_path))
return run_and_analyze(resultlog_path)
def test_full_options_caching(self) -> None:
with temporary_file_path() as config:
args = self._config_path(config)
bootstrapper = OptionsBootstrapper.create(env={}, args=args)
opts1 = bootstrapper.get_full_options(known_scope_infos=[
ScopeInfo("", ScopeInfo.GLOBAL),
ScopeInfo("foo", ScopeInfo.TASK),
])
opts2 = bootstrapper.get_full_options(known_scope_infos=[
ScopeInfo("foo", ScopeInfo.TASK),
ScopeInfo("", ScopeInfo.GLOBAL),
])
assert opts1 is opts2
opts3 = bootstrapper.get_full_options(known_scope_infos=[
ScopeInfo("", ScopeInfo.GLOBAL),
ScopeInfo("foo", ScopeInfo.TASK),
ScopeInfo("", ScopeInfo.GLOBAL),
])
assert opts1 is opts3
opts4 = bootstrapper.get_full_options(
known_scope_infos=[ScopeInfo("", ScopeInfo.GLOBAL)])
assert opts1 is not opts4
opts5 = bootstrapper.get_full_options(
known_scope_infos=[ScopeInfo("", ScopeInfo.GLOBAL)])
assert opts4 is opts5
assert opts1 is not opts5
def test_stats_local_json_file(self):
with temporary_file_path() as tmpfile:
pants_run = self.run_pants([
'test',
'--run-tracker-stats-local-json-file={}'.format(tmpfile),
'--run-tracker-stats-option-scopes-to-record=["GLOBAL", "GLOBAL^enable_pantsd", "compile.zinc^capture_classpath"]',
'testprojects/src/java/org/pantsbuild/testproject/unicode/main',
])
self.assert_success(pants_run)
with open(tmpfile, 'r') as fp:
stats_json = json.load(fp)
self.assertIn('outcomes', stats_json)
self.assertEqual(stats_json['outcomes']['main:test'], 'SUCCESS')
self.assertIn('artifact_cache_stats', stats_json)
self.assertIn('run_info', stats_json)
self.assertIn('self_timings', stats_json)
self.assertIn('cumulative_timings', stats_json)
self.assertIn('pantsd_stats', stats_json)
self.assertIn('recorded_options', stats_json)
self.assertIn('GLOBAL', stats_json['recorded_options'])
self.assertIs(stats_json['recorded_options']['GLOBAL']['enable_pantsd'], False)
self.assertEqual(stats_json['recorded_options']['GLOBAL']['level'], 'info')
self.assertIs(stats_json['recorded_options']['GLOBAL^enable_pantsd'], False)
self.assertEqual(stats_json['recorded_options']['compile.zinc^capture_classpath'], True)
def test_spaces_preservend_when_populating_zinc_args_product_from_argfile(
self):
with temporary_file_path() as arg_file_path:
compile_contexts = {
self.java_target:
CompileContext(self.java_target,
analysis_file="",
classes_dir="",
jar_file="",
log_dir="",
args_file=arg_file_path,
sources=[],
post_compile_merge_dir="")
}
task = self.get_task()
args = [
'-classpath', 'a.jar:b.jar',
'-C-Xplugin:some_javac_plugin with args'
]
task.context.products.safe_create_data(
'zinc_args', init_func=lambda: {self.java_target: []})
task.write_argsfile(compile_contexts[self.java_target], args)
task.register_extra_products_from_contexts([self.java_target],
compile_contexts)
zinc_args = task.context.products.get_data('zinc_args')[
self.java_target]
assert args == zinc_args
def test_force_override(self):
jars = list(self.a.payload.jars)
with temporary_file_path() as ivyxml:
init_subsystem(JarDependencyManagement)
IvyUtils.generate_ivy([self.a], jars=jars, excludes=[], ivyxml=ivyxml, confs=['default'])
doc = ET.parse(ivyxml).getroot()
conf = self.find_single(doc, 'configurations/conf')
self.assert_attributes(conf, name='default')
dependencies = list(doc.findall('dependencies/dependency'))
self.assertEqual(2, len(dependencies))
dep1 = dependencies[0]
self.assert_attributes(dep1, org='org1', name='name1', rev='rev1')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
dep2 = dependencies[1]
self.assert_attributes(dep2, org='org2', name='name2', rev='rev2', force='true')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
override = self.find_single(doc, 'dependencies/override')
self.assert_attributes(override, org='org2', module='name2', rev='rev2')
def execute_literal(self, scheduler, execution_request):
returns, throws = scheduler.execute(execution_request)
if throws:
with temporary_file_path(cleanup=False, suffix='.dot') as dot_file:
scheduler.visualize_graph_to_file(dot_file)
raise ValueError('At least one root failed: {}. Visualized as {}'.format(throws, dot_file))
return list(state.value for _, state in returns)
def test_stats_local_json_file_v1(self):
with temporary_file_path() as tmpfile:
pants_run = self.run_pants([
'test',
'--run-tracker-stats-local-json-file={}'.format(tmpfile),
'--run-tracker-stats-version=1',
'--run-tracker-stats-option-scopes-to-record=["GLOBAL", "GLOBAL^v2_ui", "compile.zinc^capture_classpath"]',
'testprojects/src/java/org/pantsbuild/testproject/unicode/main',
])
self.assert_success(pants_run)
with open(tmpfile, 'r') as fp:
stats_json = json.load(fp)
self.assertIn('outcomes', stats_json)
self.assertEqual(stats_json['outcomes']['main:test'], 'SUCCESS')
self.assertIn('artifact_cache_stats', stats_json)
self.assertIn('run_info', stats_json)
self.assertIn('self_timings', stats_json)
self.assertIn('cumulative_timings', stats_json)
self.assertIn('pantsd_stats', stats_json)
self.assertIn('recorded_options', stats_json)
self.assertIn('GLOBAL', stats_json['recorded_options'])
self.assertIs(stats_json['recorded_options']['GLOBAL']['v2_ui'], False)
self.assertEqual(stats_json['recorded_options']['GLOBAL']['level'], 'info')
self.assertIs(stats_json['recorded_options']['GLOBAL^v2_ui'], False)
self.assertEqual(stats_json['recorded_options']['compile.zinc^capture_classpath'], True)
def test_fetch_jar_with_url_specified(self):
# Resolve a jar with a URL, with fresh ivy cache dirs to ensure that it won't get cached.
# Run a fetch with a fresh workdir.
# The fetch should still pick up the correct file.
with self._temp_cache_dir(), temporary_file_path() as jarfile:
jar_with_url = JarDependency('org1', 'name1', rev='1234', url='file://{}'.format(jarfile))
jar_lib = self.make_target('//:a', JarLibrary, jars=[jar_with_url])
with self._temp_workdir() as workdir, self._temp_ivy_cache_dir():
self.resolve([jar_lib])
ivy_resolve_workdir = self._find_resolve_workdir(workdir)
with self._temp_workdir() as workdir, self._temp_ivy_cache_dir():
fetch_classpath = self.resolve([jar_lib])
ivy_resolve_workdir = self._find_resolve_workdir(workdir)
report_path = os.path.join(ivy_resolve_workdir, 'fetch-report-default.xml')
self._assertIsFile(report_path)
_unused_conf, lib_symlink = fetch_classpath.get_for_target(jar_lib)[0]
with open(jarfile) as jarfile_f:
with open(lib_symlink) as symlink_f:
self.assertTrue(jarfile_f.read() == symlink_f.read(),
'Expected linked jar and original to match.')
def test_full_options_caching(self):
with temporary_file_path() as config:
args = self._config_path(config)
bootstrapper = OptionsBootstrapper(env={}, args=args)
opts1 = bootstrapper.get_full_options(
known_scope_infos=[ScopeInfo("", ScopeInfo.GLOBAL), ScopeInfo("foo", ScopeInfo.TASK)]
)
opts2 = bootstrapper.get_full_options(
known_scope_infos=[ScopeInfo("foo", ScopeInfo.TASK), ScopeInfo("", ScopeInfo.GLOBAL)]
)
self.assertIs(opts1, opts2)
opts3 = bootstrapper.get_full_options(
known_scope_infos=[
ScopeInfo("", ScopeInfo.GLOBAL),
ScopeInfo("foo", ScopeInfo.TASK),
ScopeInfo("", ScopeInfo.GLOBAL),
]
)
self.assertIs(opts1, opts3)
opts4 = bootstrapper.get_full_options(known_scope_infos=[ScopeInfo("", ScopeInfo.GLOBAL)])
self.assertIsNot(opts1, opts4)
opts5 = bootstrapper.get_full_options(known_scope_infos=[ScopeInfo("", ScopeInfo.GLOBAL)])
self.assertIs(opts4, opts5)
self.assertIsNot(opts1, opts5)
def rule_subgraph_visualization(self, root_subject_types, product_type):
with temporary_file_path() as path:
self.visualize_rule_subgraph_to_file(path, root_subject_types,
product_type)
with open(path, "r") as fd:
for line in fd.readlines():
yield line.rstrip()
def graph_trace(self, execution_request):
with temporary_file_path() as path:
self._native.lib.graph_trace(self._scheduler, execution_request,
path.encode('utf-8'))
with open(path, 'r') as fd:
for line in fd.readlines():
yield line.rstrip()
def test_force_override(self):
jars = list(self.simple.payload.jars)
with temporary_file_path() as ivyxml:
self.ivy_utils._generate_ivy([self.simple], jars=jars, excludes=[], ivyxml=ivyxml, confs=["default"])
doc = ET.parse(ivyxml).getroot()
conf = self.find_single(doc, "configurations/conf")
self.assert_attributes(conf, name="default")
dependencies = list(doc.findall("dependencies/dependency"))
self.assertEqual(2, len(dependencies))
dep1 = dependencies[0]
self.assert_attributes(dep1, org="org1", name="name1", rev="rev1")
conf = self.find_single(dep1, "conf")
self.assert_attributes(conf, name="default", mapped="default")
dep2 = dependencies[1]
self.assert_attributes(dep2, org="org2", name="name2", rev="rev2", force="true")
conf = self.find_single(dep1, "conf")
self.assert_attributes(conf, name="default", mapped="default")
override = self.find_single(doc, "dependencies/override")
self.assert_attributes(override, org="org2", module="name2", rev="rev2")
def test_full_options_caching(self) -> None:
with temporary_file_path() as config:
args = self._config_path(config)
bootstrapper = OptionsBootstrapper.create(env={}, args=args, allow_pantsrc=False)
opts1 = bootstrapper.full_options_for_scopes(
known_scope_infos=[
ScopeInfo(""),
ScopeInfo("foo"),
]
)
opts2 = bootstrapper.full_options_for_scopes(
known_scope_infos=[
ScopeInfo("foo"),
ScopeInfo(""),
]
)
assert opts1 is opts2
opts3 = bootstrapper.full_options_for_scopes(
known_scope_infos=[
ScopeInfo(""),
ScopeInfo("foo"),
ScopeInfo(""),
]
)
assert opts1 is opts3
opts4 = bootstrapper.full_options_for_scopes(known_scope_infos=[ScopeInfo("")])
assert opts1 is not opts4
opts5 = bootstrapper.full_options_for_scopes(known_scope_infos=[ScopeInfo("")])
assert opts4 is opts5
assert opts1 is not opts5
def test_fetch_jar_with_url_specified(self):
# Resolve a jar with a URL, with fresh ivy cache dirs to ensure that it won't get cached.
# Run a fetch with a fresh workdir.
# The fetch should still pick up the correct file.
with self._temp_cache_dir(), temporary_file_path() as jarfile:
jar_with_url = JarDependency('org1',
'name1',
rev='1234',
url='file://{}'.format(jarfile))
jar_lib = self.make_target('//:a', JarLibrary, jars=[jar_with_url])
with self._temp_workdir() as workdir, self._temp_ivy_cache_dir():
self.resolve([jar_lib])
ivy_resolve_workdir = self._find_resolve_workdir(workdir)
with self._temp_workdir() as workdir, self._temp_ivy_cache_dir():
fetch_classpath = self.resolve([jar_lib])
ivy_resolve_workdir = self._find_resolve_workdir(workdir)
report_path = os.path.join(ivy_resolve_workdir,
'fetch-report-default.xml')
self._assertIsFile(report_path)
_unused_conf, lib_symlink = fetch_classpath.get_for_target(
jar_lib)[0]
with open(jarfile, 'r') as jarfile_f:
with open(lib_symlink, 'r') as symlink_f:
self.assertTrue(
jarfile_f.read() == symlink_f.read(),
'Expected linked jar and original to match.')
def execute_literal(self, scheduler, execution_request):
returns, throws = scheduler.execute(execution_request)
if throws:
with temporary_file_path(cleanup=False, suffix='.dot') as dot_file:
scheduler.visualize_graph_to_file(dot_file)
raise ValueError('At least one root failed: {}. Visualized as {}'.format(throws, dot_file))
return list(state.value for _, state in returns)
def test_full_options_caching(self):
with temporary_file_path() as config:
bootstrapper = OptionsBootstrapper(env={},
configpath=config,
args=[])
opts1 = bootstrapper.get_full_options(known_scope_infos=[
ScopeInfo('', ScopeInfo.GLOBAL),
ScopeInfo('foo', ScopeInfo.TASK)
])
opts2 = bootstrapper.get_full_options(known_scope_infos=[
ScopeInfo('foo', ScopeInfo.TASK),
ScopeInfo('', ScopeInfo.GLOBAL)
])
self.assertIs(opts1, opts2)
opts3 = bootstrapper.get_full_options(known_scope_infos=[
ScopeInfo('', ScopeInfo.GLOBAL),
ScopeInfo('foo', ScopeInfo.TASK),
ScopeInfo('', ScopeInfo.GLOBAL)
])
self.assertIs(opts1, opts3)
opts4 = bootstrapper.get_full_options(
known_scope_infos=[ScopeInfo('', ScopeInfo.GLOBAL)])
self.assertIsNot(opts1, opts4)
opts5 = bootstrapper.get_full_options(
known_scope_infos=[ScopeInfo('', ScopeInfo.GLOBAL)])
self.assertIs(opts4, opts5)
self.assertIsNot(opts1, opts5)
def test_force_override(self):
jars = list(self.a.payload.jars)
with temporary_file_path() as ivyxml:
init_subsystem(JarDependencyManagement)
IvyUtils.generate_ivy([self.a], jars=jars, excludes=[], ivyxml=ivyxml, confs=['default'])
doc = ET.parse(ivyxml).getroot()
conf = self.find_single(doc, 'configurations/conf')
self.assert_attributes(conf, name='default')
dependencies = list(doc.findall('dependencies/dependency'))
self.assertEqual(2, len(dependencies))
dep1 = dependencies[0]
self.assert_attributes(dep1, org='org1', name='name1', rev='rev1')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
dep2 = dependencies[1]
self.assert_attributes(dep2, org='org2', name='name2', rev='rev2', force='true')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
override = self.find_single(doc, 'dependencies/override')
self.assert_attributes(override, org='org2', module='name2', rev='rev2')
def test_force_override(self):
jars = list(self.a.payload.jars)
with temporary_file_path() as ivyxml:
init_subsystem(JarDependencyManagement)
IvyUtils.generate_ivy(
[self.a], jars=jars, excludes=[], ivyxml=ivyxml, confs=["default"]
)
doc = ET.parse(ivyxml).getroot()
conf = self.find_single(doc, "configurations/conf")
self.assert_attributes(conf, name="default")
dependencies = list(doc.findall("dependencies/dependency"))
self.assertEqual(2, len(dependencies))
dep1 = dependencies[0]
self.assert_attributes(dep1, org="org1", name="name1", rev="rev1")
conf = self.find_single(dep1, "conf")
self.assert_attributes(conf, name="default", mapped="default")
dep2 = dependencies[1]
self.assert_attributes(dep2, org="org2", name="name2", rev="rev2", force="true")
conf = self.find_single(dep1, "conf")
self.assert_attributes(conf, name="default", mapped="default")
override = self.find_single(doc, "dependencies/override")
self.assert_attributes(override, org="org2", module="name2", rev="rev2")
def graph_trace(self, session, execution_request):
with temporary_file_path() as path:
self._native.lib.graph_trace(self._scheduler, session,
execution_request, path)
with open(path, "r") as fd:
for line in fd.readlines():
yield line.rstrip()
def test(self):
init_subsystem(SourceRootConfig)
self.create_file('src/python/package/module.py')
implicit_package_target = self.make_target(spec='src/python/package',
target_type=PythonLibrary,
sources=['module.py'])
pbw = self.pex_builder_wrapper()
pbw.add_sources_from(implicit_package_target)
with temporary_file_path() as pex:
pbw.build(pex)
with self.extracted_pex(pex) as chroot_path:
# Check the paths we know about:
package_path = chroot_path / 'package'
self.assert_dir_perms(package_path)
user_files = {
package_path / f
for f in ('__init__.py', 'module.py')
}
for user_file in user_files:
self.assert_file_perms(user_file)
# And all other paths pex generates (__main__.py, PEX-INFO, .deps/, etc...):
for root, dirs, files in os.walk(chroot_path):
for d in dirs:
dir_path = pathlib.Path(root) / d
if dir_path != package_path:
self.assert_dir_perms(dir_path)
for f in files:
file_path = pathlib.Path(root) / f
if file_path not in user_files:
self.assert_file_perms(file_path)
def test_arg_scoping(self):
# Some basic smoke tests.
options = self._parse('./pants --verbose')
self.assertEqual(True, options.for_global_scope().verbose)
options = self._parse('./pants -z compile path/to/tgt')
self.assertEqual(['path/to/tgt'], options.target_specs)
self.assertEqual(True, options.for_global_scope().verbose)
with self.assertRaises(ParseError):
self._parse('./pants --unregistered-option compile').for_global_scope()
# Scoping of different values of the same option.
# Also tests the --no-* boolean flag inverses.
options = self._parse('./pants --verbose compile.java --no-verbose')
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(True, options.for_scope('compile').verbose)
self.assertEqual(False, options.for_scope('compile.java').verbose)
options = self._parse('./pants --verbose compile --no-verbose compile.java -z test '
'test.junit --no-verbose')
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(False, options.for_scope('compile').verbose)
self.assertEqual(True, options.for_scope('compile.java').verbose)
self.assertEqual(True, options.for_scope('test').verbose)
self.assertEqual(False, options.for_scope('test.junit').verbose)
# Test action=append option.
options = self._parse('./pants', config={'DEFAULT': {'y': ['88', '-99']}})
self.assertEqual([88, -99], options.for_global_scope().y)
options = self._parse('./pants --y=5 --y=-6 --y=77',
config={'DEFAULT': {'y': ['88', '-99']}})
self.assertEqual([88, -99, 5, -6, 77], options.for_global_scope().y)
options = self._parse('./pants')
self.assertEqual([], options.for_global_scope().y)
options = self._parse('./pants ', env={'PANTS_CONFIG_OVERRIDE': "['123','456']"})
self.assertEqual(['123', '456'], options.for_global_scope().config_override)
options = self._parse('./pants ', env={'PANTS_CONFIG_OVERRIDE': "['']"})
self.assertEqual([''], options.for_global_scope().config_override)
# Test list-typed option.
options = self._parse('./pants --listy=\'[1, 2]\'',
config={'DEFAULT': {'listy': '[3, 4]'}})
self.assertEqual([1, 2], options.for_global_scope().listy)
# Test dict-typed option.
options = self._parse('./pants --dicty=\'{"c": "d"}\'')
self.assertEqual({'c': 'd'}, options.for_global_scope().dicty)
# Test target_list-typed option.
options = self._parse('./pants --target_listy=\'["//:foo", "//:bar"]\'')
self.assertEqual(['//:foo', '//:bar'], options.for_global_scope().target_listy)
# Test file-typed option.
with temporary_file_path() as fp:
options = self._parse('./pants --filey="{}"'.format(fp))
self.assertEqual(fp, options.for_global_scope().filey)
def instrument(self, targets, tests, compute_junit_classpath):
junit_classpath = compute_junit_classpath()
cobertura_cp = self._task_exports.tool_classpath('cobertura-instrument')
aux_classpath = os.pathsep.join(relativize_paths(junit_classpath, get_buildroot()))
safe_delete(self._coverage_datafile)
classes_by_target = self._context.products.get_data('classes_by_target')
for target in targets:
if self.is_coverage_target(target):
classes_by_rootdir = classes_by_target.get(target)
if classes_by_rootdir:
for root, products in classes_by_rootdir.rel_paths():
self._rootdirs[root].update(products)
# Cobertura uses regular expressions for filters, and even then there are still problems
# with filtering. It turned out to be easier to just select which classes to instrument
# by filtering them here.
# TODO(ji): Investigate again how we can use cobertura's own filtering mechanisms.
if self._coverage_filters:
for basedir, classes in self._rootdirs.items():
updated_classes = []
for cls in classes:
does_match = False
for positive_filter in self._include_filters:
if fnmatch.fnmatchcase(_classfile_to_classname(cls), positive_filter):
does_match = True
for negative_filter in self._exclude_filters:
if fnmatch.fnmatchcase(_classfile_to_classname(cls), negative_filter):
does_match = False
if does_match:
updated_classes.append(cls)
self._rootdirs[basedir] = updated_classes
for basedir, classes in self._rootdirs.items():
if not classes:
continue # No point in running instrumentation if there is nothing to instrument!
self._nothing_to_instrument = False
args = [
'--basedir',
basedir,
'--datafile',
self._coverage_datafile,
'--auxClasspath',
aux_classpath,
]
with temporary_file_path(cleanup=False) as instrumented_classes_file:
with file(instrumented_classes_file, 'wb') as icf:
icf.write(('\n'.join(classes) + '\n').encode('utf-8'))
self._context.log.debug('instrumented classes in {0}'.format(instrumented_classes_file))
args.append('--listOfFilesToInstrument')
args.append(instrumented_classes_file)
main = 'net.sourceforge.cobertura.instrument.InstrumentMain'
execute_java = self.preferred_jvm_distribution_for_targets(targets).execute_java
result = execute_java(classpath=cobertura_cp,
main=main,
jvm_options=self._coverage_jvm_options,
args=args,
workunit_factory=self._context.new_workunit,
workunit_name='cobertura-instrument')
if result != 0:
raise TaskError("java {0} ... exited non-zero ({1})"
" 'failed to instrument'".format(main, result))
def _do_test_successful_request_cleans_result_dir(self, cache, key):
with self.setup_test_file(cache.artifact_root) as path:
with temporary_dir() as results_dir:
with temporary_file_path(root_dir=results_dir) as canary:
call_insert((cache, key, [path], False))
call_use_cached_files((cache, key, results_dir))
# Results content should have been deleted.
self.assertFalse(os.path.exists(canary))
def trace(self):
"""Yields a stringified 'stacktrace' starting from the scheduler's roots."""
with self._product_graph_lock:
with temporary_file_path() as path:
self._native.lib.graph_trace(self._scheduler, bytes(path))
with open(path) as fd:
for line in fd.readlines():
yield line.rstrip()
def trace(self):
"""Yields a stringified 'stacktrace' starting from the scheduler's roots."""
with self._product_graph_lock:
with temporary_file_path() as path:
self._native.lib.graph_trace(self._scheduler, bytes(path))
with open(path) as fd:
for line in fd.readlines():
yield line.rstrip()
def _do_test_successful_request_cleans_result_dir(self, cache, key):
with self.setup_test_file(cache.artifact_root) as path:
with temporary_dir() as results_dir:
with temporary_file_path(root_dir=results_dir) as canary:
map(call_insert, [(cache, key, [path], False)])
map(call_use_cached_files, [(cache, key, results_dir)])
# Results content should have been deleted.
self.assertFalse(os.path.exists(canary))
def test_stats_local_json_file_v1(self):
with temporary_file_path() as tmpfile:
pants_run = self.run_pants([
"list",
"test",
f"--run-tracker-stats-local-json-file={tmpfile}",
"--run-tracker-stats-version=1",
"--reporting-zipkin-trace-v2",
'--run-tracker-stats-option-scopes-to-record=["GLOBAL", "GLOBAL^time", "compile.rsc^capture_classpath"]',
"testprojects/src/java/org/pantsbuild/testproject/extra_jvm_options::",
"testprojects/src/java/org/pantsbuild/testproject/unicode/main",
])
self.assert_success(pants_run)
with open(tmpfile, "r") as fp:
stats_json = json.load(fp)
self.assertIn("outcomes", stats_json)
self.assertEqual(stats_json["outcomes"]["main:test"],
"SUCCESS")
self.assertIn("artifact_cache_stats", stats_json)
self.assertIn("run_info", stats_json)
computed_goals = stats_json["run_info"]["computed_goals"]
self.assertIsInstance(computed_goals, list)
# Explicit v1 goal on the command line:
self.assertIn("test", stats_json["run_info"]["computed_goals"])
# v1 goal implied by dependencies between goals:
self.assertIn("compile",
stats_json["run_info"]["computed_goals"])
# Check that v2 goals are included:
self.assertIn("list", stats_json["run_info"]["computed_goals"])
# Expanded to canonical form, but not expanded to its actual targets.
self.assertEquals(
[
"testprojects/src/java/org/pantsbuild/testproject/extra_jvm_options::",
"testprojects/src/java/org/pantsbuild/testproject/unicode/main:main",
],
stats_json["run_info"]["specs_from_command_line"],
)
self.assertIn("self_timings", stats_json)
self.assertIn("cumulative_timings", stats_json)
self.assertIn("pantsd_stats", stats_json)
self.assertIn("recorded_options", stats_json)
self.assertIn("GLOBAL", stats_json["recorded_options"])
self.assertNotIn("engine_workunits",
stats_json["pantsd_stats"])
self.assertIs(stats_json["recorded_options"]["GLOBAL"]["time"],
False)
self.assertEqual(
stats_json["recorded_options"]["GLOBAL"]["level"], "info")
self.assertIs(stats_json["recorded_options"]["GLOBAL^time"],
False)
self.assertEqual(
stats_json["recorded_options"]
["compile.rsc^capture_classpath"], True)
def instrument(self, targets, tests, compute_junit_classpath):
junit_classpath = compute_junit_classpath()
cobertura_cp = self._task_exports.tool_classpath('cobertura-instrument')
aux_classpath = os.pathsep.join(relativize_paths(junit_classpath, get_buildroot()))
safe_delete(self._coverage_datafile)
classes_by_target = self._context.products.get_data('classes_by_target')
for target in targets:
if self.is_coverage_target(target):
classes_by_rootdir = classes_by_target.get(target)
if classes_by_rootdir:
for root, products in classes_by_rootdir.rel_paths():
self._rootdirs[root].update(products)
# Cobertura uses regular expressions for filters, and even then there are still problems
# with filtering. It turned out to be easier to just select which classes to instrument
# by filtering them here.
# TODO(ji): Investigate again how we can use cobertura's own filtering mechanisms.
if self._coverage_filters:
for basedir, classes in self._rootdirs.items():
updated_classes = []
for cls in classes:
does_match = False
for positive_filter in self._include_filters:
if fnmatch.fnmatchcase(_classfile_to_classname(cls), positive_filter):
does_match = True
for negative_filter in self._exclude_filters:
if fnmatch.fnmatchcase(_classfile_to_classname(cls), negative_filter):
does_match = False
if does_match:
updated_classes.append(cls)
self._rootdirs[basedir] = updated_classes
for basedir, classes in self._rootdirs.items():
if not classes:
continue # No point in running instrumentation if there is nothing to instrument!
self._nothing_to_instrument = False
args = [
'--basedir',
basedir,
'--datafile',
self._coverage_datafile,
'--auxClasspath',
aux_classpath,
]
with temporary_file_path(cleanup=False) as instrumented_classes_file:
with file(instrumented_classes_file, 'wb') as icf:
icf.write(('\n'.join(classes) + '\n').encode('utf-8'))
self._context.log.debug('instrumented classes in {0}'.format(instrumented_classes_file))
args.append('--listOfFilesToInstrument')
args.append(instrumented_classes_file)
main = 'net.sourceforge.cobertura.instrument.InstrumentMain'
result = execute_java(classpath=cobertura_cp,
main=main,
jvm_options=self._coverage_jvm_options,
args=args,
workunit_factory=self._context.new_workunit,
workunit_name='cobertura-instrument')
if result != 0:
raise TaskError("java {0} ... exited non-zero ({1})"
" 'failed to instrument'".format(main, result))
def test_stats_local_json_file_v1(self):
with temporary_file_path() as tmpfile:
pants_run = self.run_pants([
'list',
'test',
f'--run-tracker-stats-local-json-file={tmpfile}',
'--run-tracker-stats-version=1',
'--reporting-zipkin-trace-v2',
'--run-tracker-stats-option-scopes-to-record=["GLOBAL", "GLOBAL^time", "compile.rsc^capture_classpath"]',
'testprojects/src/java/org/pantsbuild/testproject/extra_jvm_options::',
'testprojects/src/java/org/pantsbuild/testproject/unicode/main',
])
self.assert_success(pants_run)
with open(tmpfile, 'r') as fp:
stats_json = json.load(fp)
self.assertIn('outcomes', stats_json)
self.assertEqual(stats_json['outcomes']['main:test'],
'SUCCESS')
self.assertIn('artifact_cache_stats', stats_json)
self.assertIn('run_info', stats_json)
computed_goals = stats_json['run_info']['computed_goals']
self.assertIsInstance(computed_goals, list)
# Explicit v1 goal on the command line:
self.assertIn('test', stats_json['run_info']['computed_goals'])
# v1 goal implied by dependencies between goals:
self.assertIn('compile',
stats_json['run_info']['computed_goals'])
# Check that v2 goals are included:
self.assertIn('list', stats_json['run_info']['computed_goals'])
# Expanded to canonical form, but not expanded to its actual targets.
self.assertEquals(
[
'testprojects/src/java/org/pantsbuild/testproject/extra_jvm_options::',
'testprojects/src/java/org/pantsbuild/testproject/unicode/main:main',
],
stats_json['run_info']['specs_from_command_line'],
)
self.assertIn('self_timings', stats_json)
self.assertIn('cumulative_timings', stats_json)
self.assertIn('pantsd_stats', stats_json)
self.assertIn('recorded_options', stats_json)
self.assertIn('GLOBAL', stats_json['recorded_options'])
self.assertNotIn('engine_workunits',
stats_json['pantsd_stats'])
self.assertIs(stats_json['recorded_options']['GLOBAL']['time'],
False)
self.assertEqual(
stats_json['recorded_options']['GLOBAL']['level'], 'info')
self.assertIs(stats_json['recorded_options']['GLOBAL^time'],
False)
self.assertEqual(
stats_json['recorded_options']
['compile.rsc^capture_classpath'], True)
def execute(self):
mypy_interpreter = self.find_mypy_interpreter()
if not mypy_interpreter:
raise TaskError(
'Unable to find a Python {} interpreter (required for mypy).'.
format(self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT))
sources = self._calculate_python_sources(self.context.target_roots)
if not sources:
self.context.log.debug('No Python sources to check.')
return
# Determine interpreter used by the sources so we can tell mypy.
interpreter_for_targets = self._interpreter_cache.select_interpreter_for_targets(
self.context.target_roots)
if not interpreter_for_targets:
raise TaskError(
'No Python interpreter compatible with specified sources.')
with temporary_file_path() as sources_list_path:
with open(sources_list_path, 'w') as f:
for source in sources:
f.write('{}\n'.format(source))
# Construct the mypy command line.
cmd = [
'--python-version={}'.format(
interpreter_for_targets.identity.python)
]
if self.get_options().config_file:
cmd.append('--config-file={}'.format(
os.path.join(get_buildroot(),
self.get_options().config_file)))
cmd.extend(self.get_passthru_args())
cmd.append('@{}'.format(sources_list_path))
self.context.log.debug('mypy command: {}'.format(' '.join(cmd)))
# Collect source roots for the targets being checked.
source_roots = self._collect_source_roots()
mypy_path = os.pathsep.join(
[os.path.join(get_buildroot(), root) for root in source_roots])
# Execute mypy.
with self.context.new_workunit(
name='check',
labels=[WorkUnitLabel.TOOL, WorkUnitLabel.RUN],
log_config=WorkUnit.LogConfig(
level=self.get_options().level,
colors=self.get_options().colors),
cmd=' '.join(cmd)) as workunit:
returncode = self._run_mypy(mypy_interpreter,
cmd,
env={'MYPYPATH': mypy_path},
stdout=workunit.output('stdout'),
stderr=subprocess.STDOUT)
if returncode != 0:
raise TaskError('mypy failed: code={}'.format(returncode))
def try_insert(self, cache_key, paths):
with temporary_file_path() as tarfile:
artifact = TarballArtifact(self.artifact_root, tarfile, self.compress)
artifact.collect(paths)
with open(tarfile, 'rb') as infile:
remote_path = self._remote_path_for_key(cache_key)
if not self._request('PUT', remote_path, body=infile):
raise self.CacheError('Failed to PUT to %s. Error: 404' % self._url_string(remote_path))
def test_rejects_binary(self):
with temporary_file_path() as p:
fb = FileBackedRWBuf(p)
sw = StringWriter(fb)
try:
with self.assertRaises(ValueError):
sw.write(b"Curious Zelda")
finally:
fb.close()
def test_resolve_with_local_url(self):
with temporary_file_path(suffix='.jar') as url:
dep_with_url = JarDependency('commons-lang', 'commons-lang', '2.5', url='file://' + url)
dep_with_url_lib = self.make_target('//:a', JarLibrary, jars=[dep_with_url])
compile_classpath = self.resolve([dep_with_url_lib])
# Get paths on compile classpath and assert that it starts with '.../coursier/cache/absolute'
paths = [tup[1] for tup in compile_classpath.get_for_target(dep_with_url_lib)]
self.assertTrue(any(self._cache_dir_regex('absolute').search(path) for path in paths), str(paths))
def visualize_execution_graph(execution_graph):
with temporary_file() as fp:
for line in create_digraph(execution_graph):
fp.write(line)
fp.write("\n")
fp.close()
with temporary_file_path(cleanup=False) as image_file:
subprocess.check_call("dot -Tsvg -o{} {}".format(image_file, fp.name), shell=True)
binary_util.ui_open(image_file)
def test_arg_scoping(self):
# Some basic smoke tests.
options = self._parse('./pants --verbose')
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(True, options.for_global_scope().v)
options = self._parse('./pants -v compile path/to/tgt')
self.assertEqual(['path/to/tgt'], options.target_specs)
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(True, options.for_global_scope().v)
# Scoping of different values of the same option.
# Also tests the --no-* boolean flag inverses.
options = self._parse('./pants --verbose compile.java --no-verbose')
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(True, options.for_scope('compile').verbose)
self.assertEqual(False, options.for_scope('compile.java').verbose)
options = self._parse('./pants --verbose compile --no-verbose compile.java -v test '
'test.junit --no-verbose')
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(False, options.for_scope('compile').verbose)
self.assertEqual(True, options.for_scope('compile.java').verbose)
self.assertEqual(True, options.for_scope('test').verbose)
self.assertEqual(False, options.for_scope('test.junit').verbose)
# Proper shadowing of a re-registered flag. The flag's -x alias retains its old meaning.
options = self._parse('./pants --no-xlong test --xlong=100 -x')
self.assertEqual(False, options.for_global_scope().xlong)
self.assertEqual(False, options.for_global_scope().x)
self.assertEqual(100, options.for_scope('test').xlong)
self.assertEqual(True, options.for_scope('test').x)
# Test action=append option.
options = self._parse('./pants', config={'DEFAULT': {'y': ['88', '-99']}})
self.assertEqual([88, -99], options.for_global_scope().y)
options = self._parse('./pants --y=5 --y=-6 --y=77',
config={'DEFAULT': {'y': ['88', '-99']}})
self.assertEqual([88, -99, 5, -6, 77], options.for_global_scope().y)
# Test list-typed option.
options = self._parse('./pants --listy=\'["c", "d"]\'',
config={'DEFAULT': {'listy': ["a", "b"]}})
self.assertEqual(['c', 'd'], options.for_global_scope().listy)
# Test dict-typed option.
options = self._parse('./pants --dicty=\'{"c": "d"}\'')
self.assertEqual({'c': 'd'}, options.for_global_scope().dicty)
# Test target_list-typed option.
options = self._parse('./pants --target_listy=\'["//:foo", "//:bar"]\'')
self.assertEqual(['//:foo', '//:bar'], options.for_global_scope().target_listy)
# Test file-typed option.
with temporary_file_path() as fp:
options = self._parse('./pants --filey="{}"'.format(fp))
self.assertEqual(fp, options.for_global_scope().filey)
def execute(self, scheduler, product, *subjects):
"""Runs an ExecutionRequest for the given product and subjects, and returns the result value."""
request = self.execute_request(scheduler, product, *subjects)
states = scheduler.root_entries(request).values()
if any(type(state) is not Return for state in states):
with temporary_file_path(cleanup=False, suffix='.dot') as dot_file:
scheduler.visualize_graph_to_file(request.roots, dot_file)
raise ValueError('At least one request failed: {}. Visualized as {}'.format(states, dot_file))
return list(state.value for state in states)
def test_random_estimator(self):
seedValue = 5
# The number chosen for seedValue doesn't matter, so long as it is the same for the call to
# generate a random test number and the call to create_size_estimators.
random.seed(seedValue)
rand = random.randint(0, 10000)
random.seed(seedValue)
with temporary_file_path() as src:
self.assertEqual(create_size_estimators()['random']([src]), rand)
def test_resolve_with_local_url(self):
with temporary_file_path(suffix='.jar') as url:
dep_with_url = JarDependency('commons-lang', 'commons-lang', '2.5', url='file://' + url)
dep_with_url_lib = self.make_target('//:a', JarLibrary, jars=[dep_with_url])
compile_classpath = self.resolve([dep_with_url_lib])
# Get paths on compile classpath and assert that it starts with '.../coursier/cache/absolute'
paths = [tup[1] for tup in compile_classpath.get_for_target(dep_with_url_lib)]
self.assertTrue(any(self._cache_dir_regex('absolute').search(path) for path in paths), str(paths))
def test_random_estimator(self):
seedValue = 5
# The number chosen for seedValue doesn't matter, so long as it is the same for the call to
# generate a random test number and the call to create_size_estimators.
random.seed(seedValue)
rand = random.randint(0, 10000)
random.seed(seedValue)
with temporary_file_path() as src:
self.assertEqual(create_size_estimators()['random']([src]), rand)
def visualize_execution_graph(scheduler):
with temporary_file() as fp:
for line in create_digraph(scheduler):
fp.write(line)
fp.write('\n')
fp.close()
with temporary_file_path(cleanup=False, suffix='.svg') as image_file:
subprocess.check_call('dot -Tsvg -o{} {}'.format(image_file, fp.name), shell=True)
binary_util.ui_open(image_file)
def test_arg_scoping(self):
# Some basic smoke tests.
options = self._parse('./pants --verbose')
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(True, options.for_global_scope().v)
options = self._parse('./pants -v compile tgt')
self.assertEqual(['tgt'], options.target_specs)
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(True, options.for_global_scope().v)
# Scoping of different values of the same option.
# Also tests the --no-* boolean flag inverses.
options = self._parse('./pants --verbose compile.java --no-verbose')
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(True, options.for_scope('compile').verbose)
self.assertEqual(False, options.for_scope('compile.java').verbose)
options = self._parse('./pants --verbose compile --no-verbose compile.java -v test '
'test.junit --no-verbose')
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(False, options.for_scope('compile').verbose)
self.assertEqual(True, options.for_scope('compile.java').verbose)
self.assertEqual(True, options.for_scope('test').verbose)
self.assertEqual(False, options.for_scope('test.junit').verbose)
# Proper shadowing of a re-registered flag. The flag's -x alias retains its old meaning.
options = self._parse('./pants --no-xlong test --xlong=100 -x')
self.assertEqual(False, options.for_global_scope().xlong)
self.assertEqual(False, options.for_global_scope().x)
self.assertEqual(100, options.for_scope('test').xlong)
self.assertEqual(True, options.for_scope('test').x)
# Test action=append option.
options = self._parse('./pants', config={ 'DEFAULT': { 'y': ['88', '-99'] }})
self.assertEqual([88, -99], options.for_global_scope().y)
options = self._parse('./pants --y=5 --y=-6 --y=77',
config={ 'DEFAULT': { 'y': ['88', '-99'] }})
self.assertEqual([88, -99, 5, -6, 77], options.for_global_scope().y)
# Test list-typed option.
options = self._parse('./pants --listy=\'["c", "d"]\'',
config={ 'DEFAULT': {'listy': ["a", "b"] }})
self.assertEqual(['c', 'd'], options.for_global_scope().listy)
# Test dict-typed option.
options = self._parse('./pants --dicty=\'{"c": "d"}\'')
self.assertEqual({'c': 'd'}, options.for_global_scope().dicty)
# Test target_list-typed option.
options = self._parse('./pants --target_listy=\'["//:foo", "//:bar"]\'')
self.assertEqual(['//:foo', '//:bar'], options.for_global_scope().target_listy)
# Test file-typed option.
with temporary_file_path() as fp:
options = self._parse('./pants --filey="{}"'.format(fp))
self.assertEqual(fp, options.for_global_scope().filey)
def test_write_run_info(self):
with temporary_file_path() as tmppath:
ri = RunInfo(tmppath)
ri.add_info('key1', 'val1')
ri.add_infos(('key2', ' val2'), (' key3 ', 'val3 '))
self.assertEquals({'key1': 'val1', 'key2': 'val2', 'key3': 'val3'}, ri.get_as_dict())
with open(tmppath, 'r') as tmpfile:
contents = tmpfile.read()
self.assertEquals('key1: val1\nkey2: val2\nkey3: val3\n', contents)
def test_write_stats_to_json_file(self):
# Set up
stats = {"stats": {"foo": "bar", "baz": 42}}
# Execute & verify
with temporary_file_path() as file_name:
RunTracker.write_stats_to_json(file_name, stats)
with open(file_name, "r") as f:
result = json.load(f)
self.assertEqual(stats, result)
def test_write_stats_to_json_file(self):
# Set up
stats = {'stats': {'foo': 'bar', 'baz': 42}}
# Execute & verify
with temporary_file_path() as file_name:
self.assertTrue(RunTracker.write_stats_to_json(file_name, stats))
with open(file_name) as f:
result = json.load(f)
self.assertEquals(stats, result)
def test_write_run_info(self):
with temporary_file_path() as tmppath:
ri = RunInfo(tmppath)
ri.add_info('key1', 'val1')
ri.add_infos(('key2', ' val2'), (' key3 ', 'val3 '))
self.assertEqual({'key1': 'val1', 'key2': 'val2', 'key3': 'val3'}, ri.get_as_dict())
with open(tmppath, 'r') as tmpfile:
contents = tmpfile.read()
self.assertEqual('key1: val1\nkey2: val2\nkey3: val3\n', contents)
def test_write_stats_to_json_file(self):
# Set up
stats = {'stats': {'foo': 'bar', 'baz': 42}}
# Execute & verify
with temporary_file_path() as file_name:
self.assertTrue(RunTracker.write_stats_to_json(file_name, stats))
with open(file_name) as f:
result = json.load(f)
self.assertEquals(stats, result)
def execute_literal(self, scheduler, execution_request):
result = scheduler.execute(execution_request)
if result.error:
raise result.error
states = [state for _, state in result.root_products]
if any(type(state) is not Return for state in states):
with temporary_file_path(cleanup=False, suffix='.dot') as dot_file:
scheduler.visualize_graph_to_file(dot_file)
raise ValueError('At least one root failed: {}. Visualized as {}'.format(states, dot_file))
return list(state.value for state in states)
def visualize_execution_graph(scheduler, storage, request):
with temporary_file(cleanup=False, suffix='.dot') as fp:
for line in create_digraph(scheduler, storage, request):
fp.write(line)
fp.write('\n')
print('dot file saved to: {}'.format(fp.name))
with temporary_file_path(cleanup=False, suffix='.svg') as image_file:
subprocess.check_call('dot -Tsvg -o{} {}'.format(image_file, fp.name), shell=True)
print('svg file saved to: {}'.format(image_file))
binary_util.ui_open(image_file)
def test_writes_string(self):
with temporary_file_path() as p:
fb = FileBackedRWBuf(p)
try:
sw = StringWriter(fb)
sw.write("\u2764 Curious Zelda")
finally:
fb.close()
with open(p, 'rb') as f:
contents = f.read()
self.assertEquals(contents, b'\xe2\x9d\xa4 Curious Zelda')
def _do_run_tests(self, targets, workunit):
def _extract_resultlog_filename(args):
resultlogs = [arg[arg.find('=') + 1:] for arg in args if arg.startswith('--resultlog=')]
if resultlogs:
return resultlogs[0]
else:
try:
return args[args.index('--resultlog') + 1]
except IndexError:
self.context.log.error('--resultlog specified without an argument')
return None
except ValueError:
return None
if not targets:
return PythonTestResult.rc(0)
rel_sources = list(itertools.chain(*[t.sources_relative_to_source_root() for t in targets]))
if not rel_sources:
return PythonTestResult.rc(0)
source_root = self.context.products.get_data(GatherSources.PYTHON_SOURCES).path()
sources = [os.path.join(source_root, p) for p in rel_sources]
with self._test_runner(targets, workunit) as (pex, test_args):
def run_and_analyze(resultlog_path):
result = self._do_run_tests_with_args(pex, workunit, args)
failed_targets = self._get_failed_targets_from_resultlogs(resultlog_path, targets)
return result.with_failed_targets(failed_targets)
# N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the
# top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs)
# from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726
args = ['--confcutdir', get_buildroot()]
if self.get_options().fail_fast:
args.extend(['-x'])
if self._debug:
args.extend(['-s'])
if self.get_options().colors:
args.extend(['--color', 'yes'])
for options in self.get_options().options + self.get_passthru_args():
args.extend(safe_shlex_split(options))
args.extend(test_args)
args.extend(sources)
# The user might have already specified the resultlog option. In such case, reuse it.
resultlog_arg = _extract_resultlog_filename(args)
if resultlog_arg:
return run_and_analyze(resultlog_arg)
else:
with temporary_file_path() as resultlog_path:
args.insert(0, '--resultlog={0}'.format(resultlog_path))
return run_and_analyze(resultlog_path)
def cached_chroot(self):
python_task = self.create_task(self.context(target_roots=[self.binary]))
interpreter = python_task.select_interpreter_for_targets(self.binary.closure())
pex_info = self.binary.pexinfo
platforms = self.binary.platforms
with python_task.cached_chroot(interpreter, pex_info, [self.binary], platforms) as chroot:
with temporary_file_path() as pex:
chroot.dump()
chroot.package_pex(pex)
yield chroot, pex
def rule_subgraph_visualization(self, root_subject_type, product_type):
root_type_id = TypeId(self._to_id(root_subject_type))
product_type_id = TypeConstraint(self._to_id(constraint_for(product_type)))
with temporary_file_path() as path:
self._native.lib.rule_subgraph_visualize(
self._scheduler,
root_type_id,
product_type_id,
bytes(path))
with open(path) as fd:
for line in fd.readlines():
yield line.rstrip()
def rule_subgraph_visualization(self, root_subject_type, product_type):
root_type_id = TypeId(self._to_id(root_subject_type))
product_type_id = TypeId(self._to_id(product_type))
with temporary_file_path() as path:
self._native.lib.rule_subgraph_visualize(
self._scheduler,
root_type_id,
product_type_id,
path.encode('utf-8'))
with open(path, 'r') as fd:
for line in fd.readlines():
yield line.rstrip()
def test_arg_scoping(self):
# Some basic smoke tests.
options = self._parse("./pants --verbose")
self.assertEqual(True, options.for_global_scope().verbose)
options = self._parse("./pants -v compile path/to/tgt")
self.assertEqual(["path/to/tgt"], options.target_specs)
self.assertEqual(True, options.for_global_scope().verbose)
# Scoping of different values of the same option.
# Also tests the --no-* boolean flag inverses.
options = self._parse("./pants --verbose compile.java --no-verbose")
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(True, options.for_scope("compile").verbose)
self.assertEqual(False, options.for_scope("compile.java").verbose)
options = self._parse("./pants --verbose compile --no-verbose compile.java -v test " "test.junit --no-verbose")
self.assertEqual(True, options.for_global_scope().verbose)
self.assertEqual(False, options.for_scope("compile").verbose)
self.assertEqual(True, options.for_scope("compile.java").verbose)
self.assertEqual(True, options.for_scope("test").verbose)
self.assertEqual(False, options.for_scope("test.junit").verbose)
# Test action=append option.
options = self._parse("./pants", config={"DEFAULT": {"y": ["88", "-99"]}})
self.assertEqual([88, -99], options.for_global_scope().y)
options = self._parse("./pants --y=5 --y=-6 --y=77", config={"DEFAULT": {"y": ["88", "-99"]}})
self.assertEqual([88, -99, 5, -6, 77], options.for_global_scope().y)
options = self._parse("./pants ", env={"PANTS_CONFIG_OVERRIDE": "['123','456']"})
self.assertEqual(["123", "456"], options.for_global_scope().config_override)
options = self._parse("./pants ", env={"PANTS_CONFIG_OVERRIDE": "['']"})
self.assertEqual([""], options.for_global_scope().config_override)
# Test list-typed option.
options = self._parse('./pants --listy=\'["c", "d"]\'', config={"DEFAULT": {"listy": '["a", "b"]'}})
self.assertEqual(["c", "d"], options.for_global_scope().listy)
# Test dict-typed option.
options = self._parse('./pants --dicty=\'{"c": "d"}\'')
self.assertEqual({"c": "d"}, options.for_global_scope().dicty)
# Test target_list-typed option.
options = self._parse('./pants --target_listy=\'["//:foo", "//:bar"]\'')
self.assertEqual(["//:foo", "//:bar"], options.for_global_scope().target_listy)
# Test file-typed option.
with temporary_file_path() as fp:
options = self._parse('./pants --filey="{}"'.format(fp))
self.assertEqual(fp, options.for_global_scope().filey)
