def visualize_build_request(build_root, goals, subjects): native = init_native() scheduler = setup_json_scheduler(build_root, native) execution_request = scheduler.build_request(goals, subjects) # NB: Calls `schedule` independently of `execute`, in order to render a graph before validating it. scheduler.schedule(execution_request) visualize_execution_graph(scheduler)
def create_scheduler(self, rule_index):
native = init_native()
scheduler = WrappedNativeScheduler(
native=native,
build_root='/tmp',
work_dir='/tmp/.pants.d',
ignore_patterns=tuple(),
rule_index=rule_index)
return scheduler
def _init_engine(cls):
if cls._scheduler is not None:
return
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=None,
workdir=cls._pants_workdir(),
build_file_imports_behavior='allow',
native=init_native(),
build_configuration=cls.build_config(),
build_ignore_patterns=None,
).new_session()
cls._scheduler = graph_session.scheduler_session
cls._build_graph, cls._address_mapper = graph_session.create_build_graph(
TargetRoots([]), cls._build_root()
)
def _init_engine(cls):
if cls._scheduler is not None:
return
cls._local_store_dir = os.path.realpath(safe_mkdtemp())
safe_mkdir(cls._local_store_dir)
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=None,
workdir=cls._pants_workdir(),
local_store_dir=cls._local_store_dir,
build_file_imports_behavior='allow',
native=init_native(),
options_bootstrapper=OptionsBootstrapper.create(args=['--pants-config-files=[]']),
build_configuration=cls.build_config(),
build_ignore_patterns=None,
).new_session(zipkin_trace_v2=False)
cls._scheduler = graph_session.scheduler_session
cls._build_graph, cls._address_mapper = graph_session.create_build_graph(
TargetRoots([]), cls._build_root()
)
class SchedulerTest(unittest.TestCase):
_native = init_native()
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples',
'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root, self._native)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse(
'src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse(
'src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse(
'src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse(
'3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse(
'3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def tearDown(self):
super(SchedulerTest, self).tearDown()
# Without eagerly dropping this reference, each instance created for a test method
# will live until all tests in this class have completed: can confirm by editing
# the `scheduler_destroy` call in `src/python/pants/engine/native.py`.
self.scheduler = None
def parse_specs(self, *specs):
return Specs(tuple(
self.spec_parser.parse_spec(spec) for spec in specs))
def assert_select_for_subjects(self,
walk,
selector,
subjects,
variants=None):
raise ValueError(walk)
def build(self, execution_request):
"""Execute the given request and return roots as a list of ((subject, product), value) tuples."""
result = self.scheduler.execute(execution_request)
self.assertIsNone(result.error)
return result.root_products
def request(self, products, *subjects):
return self.scheduler.execution_request(products, subjects)
def assert_root(self, root, subject, return_value):
"""Asserts that the given root has the given result."""
self.assertEquals(subject, root[0][0])
self.assertEquals(Return(return_value), root[1])
def assert_root_failed(self, root, subject, msg_str):
"""Asserts that the root was a Throw result containing the given msg string."""
self.assertEquals(subject, root[0][0])
self.assertEquals(Throw, type(root[1]))
self.assertIn(msg_str, str(root[1].exc))
def test_compile_only_3rdparty(self):
build_request = self.request([Classpath], self.guava)
root, = self.build(build_request)
self.assert_root(root, self.guava, Classpath(creator='ivy_resolve'))
@unittest.skip('Skipped to expedite landing #3821; see: #4027.')
def test_compile_only_3rdparty_internal(self):
build_request = self.request([Classpath], '3rdparty/jvm:guava')
root, = self.build(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Select(Jar), [self.guava])
self.assert_select_for_subjects(walk, Select(Classpath), [self.guava])
@unittest.skip('Skipped to expedite landing #3821; see: #4020.')
def test_gen(self):
build_request = self.request([GenGoal], self.thrift)
root, = self.build(build_request)
# Root: expect the synthetic GenGoal product.
self.assert_root(
root, self.thrift,
GenGoal("non-empty input to satisfy the Goal constructor"))
variants = {'thrift': 'apache_java'}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk,
Select(ThriftSources), [self.thrift],
variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk,
SelectVariant(
ApacheThriftJavaConfiguration,
variant_key='thrift'),
[self.thrift],
variants=variants)
@unittest.skip('Skipped to expedite landing #3821; see: #4020.')
def test_codegen_simple(self):
build_request = self.request([Classpath], self.java)
root, = self.build(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [self.guava, self.java, self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift',
name='libthrift',
rev='0.9.2',
type_alias='jar'),
Jar(org='commons-lang',
name='commons-lang',
rev='2.5',
type_alias='jar'),
Address.parse('src/thrift:slf4j-api')
]
# Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
self.assert_root(root, self.java, Classpath(creator='javac'))
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Select(Classpath), subjects)
self.assert_select_for_subjects(walk,
Select(Classpath),
variant_subjects,
variants={'thrift': 'apache_java'})
def test_consumes_resources(self):
build_request = self.request([Classpath], self.consumes_resources)
root, = self.build(build_request)
self.assert_root(root, self.consumes_resources,
Classpath(creator='javac'))
@unittest.skip('Skipped to expedite landing #3821; see: #4027.')
def test_consumes_resources_internal(self):
build_request = self.request([Classpath], self.consumes_resources)
root, = self.build(build_request)
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources, self.consumes_resources, self.guava]
self.assert_select_for_subjects(walk, Select(Classpath), subjects)
@unittest.skip('Skipped to expedite landing #3821; see: #4020.')
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request([Classpath],
self.consumes_managed_thirdparty)
root, = self.build(build_request)
# Validate the root.
self.assert_root(root, self.consumes_managed_thirdparty,
Classpath(creator='javac'))
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava, self.managed_hadoop]
self.assert_select_for_subjects(walk, Select(Classpath),
[self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk,
Select(Classpath),
managed_jars,
variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals(
{
Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')
}, {ret.value
for node, ret in walk if node.product == Jar})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request([Classpath], self.inferred_deps)
root, = self.build(build_request)
self.assert_root(root, self.inferred_deps, Classpath(creator='scalac'))
@unittest.skip('Skipped to expedite landing #3821; see: #4027.')
def test_dependency_inference_internal(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request([Classpath], self.inferred_deps)
root, = self.build(build_request)
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Select(Classpath),
[self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request([Classpath], self.java_multi)
root, = self.build(build_request)
# Validate that the root failed.
self.assert_root_failed(root, self.java_multi,
"Conflicting values produced for")
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
specs = self.parse_specs('3rdparty/jvm::')
build_request = self.scheduler.execution_request([BuildFileAddresses],
[specs])
((subject, _), root), = self.build(build_request)
# Validate the root.
self.assertEqual(specs, subject)
self.assertEqual(BuildFileAddresses, type(root.value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root.value.dependencies)
self.assertIn(self.managed_guava, root.value.dependencies)
self.assertIn(self.managed_resolve_latest, root.value.dependencies)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
specs = self.parse_specs('3rdparty/jvm:')
build_request = self.scheduler.execution_request([BuildFileAddresses],
[specs])
((subject, _), root), = self.build(build_request)
# Validate the root.
self.assertEqual(specs, subject)
self.assertEqual(BuildFileAddresses, type(root.value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root.value.dependencies)
# And that a subdirectory address is not.
self.assertNotIn(self.managed_guava, root.value.dependencies)
def test_scheduler_visualize(self):
specs = self.parse_specs('3rdparty/jvm::')
build_request = self.request([BuildFileAddresses], specs)
self.build(build_request)
with temporary_dir() as td:
output_path = os.path.join(td, 'output.dot')
self.scheduler.visualize_graph_to_file(output_path)
with open(output_path, 'rb') as fh:
graphviz_output = fh.read().strip()
self.assertIn('digraph', graphviz_output)
self.assertIn(' -> ', graphviz_output)
class GraphTestBase(unittest.TestCase):
_native = init_native()
def _make_setup_args(self, specs):
options = mock.Mock(target_specs=specs)
options.for_scope.return_value = mock.Mock(diffspec=None,
changes_since=None)
options.for_global_scope.return_value = mock.Mock(owner_of=None)
return options
def _default_build_config(self,
options_bootstrapper,
build_file_aliases=None):
# TODO: Get default BuildFileAliases by extending BaseTest post
# https://github.com/pantsbuild/pants/issues/4401
build_config = BuildConfigInitializer.get(options_bootstrapper)
if build_file_aliases:
build_config.register_aliases(build_file_aliases)
return build_config
@contextmanager
def graph_helper(self,
build_configuration=None,
build_file_imports_behavior='allow',
include_trace_on_error=True,
path_ignore_patterns=None):
with temporary_dir() as work_dir:
with temporary_dir() as local_store_dir:
path_ignore_patterns = path_ignore_patterns or []
options_bootstrapper = OptionsBootstrapper.create()
build_config = build_configuration or self._default_build_config(
options_bootstrapper)
# TODO: This test should be swapped to using TestBase.
graph_helper = EngineInitializer.setup_legacy_graph_extended(
path_ignore_patterns,
work_dir,
local_store_dir,
build_file_imports_behavior,
options_bootstrapper=options_bootstrapper,
build_configuration=build_config,
native=self._native,
include_trace_on_error=include_trace_on_error)
yield graph_helper
@contextmanager
def open_scheduler(self, specs, build_configuration=None):
with self.graph_helper(
build_configuration=build_configuration) as graph_helper:
graph, target_roots = self.create_graph_from_specs(
graph_helper, specs)
addresses = tuple(graph.inject_roots_closure(target_roots))
yield graph, addresses, graph_helper.scheduler.new_session()
def create_graph_from_specs(self, graph_helper, specs):
Subsystem.reset()
session = graph_helper.new_session()
target_roots = self.create_target_roots(specs, session,
session.symbol_table)
graph = session.create_build_graph(target_roots)[0]
return graph, target_roots
def create_target_roots(self, specs, session, symbol_table):
return TargetRootsCalculator.create(self._make_setup_args(specs),
session, symbol_table)
class SchedulerTestBase(object):
"""A mixin for classes (tests, presumably) which need to create temporary schedulers.
TODO: In the medium term, this should be part of pants_test.base_test.BaseTest.
"""
_native = init_native()
def _create_work_dir(self):
work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, work_dir)
return work_dir
def mk_fs_tree(self,
build_root_src=None,
ignore_patterns=None,
work_dir=None):
"""Create a temporary FilesystemProjectTree.
:param build_root_src: Optional directory to pre-populate from; otherwise, empty.
:returns: A FilesystemProjectTree.
"""
work_dir = work_dir or self._create_work_dir()
build_root = os.path.join(work_dir, 'build_root')
if build_root_src is not None:
shutil.copytree(build_root_src, build_root, symlinks=True)
else:
os.makedirs(build_root)
return FileSystemProjectTree(build_root,
ignore_patterns=ignore_patterns)
def mk_scheduler(self,
rules=None,
project_tree=None,
work_dir=None,
include_trace_on_error=True):
"""Creates a Scheduler with the given Rules installed."""
rules = rules or []
goals = {}
work_dir = work_dir or self._create_work_dir()
project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir)
return LocalScheduler(work_dir,
goals,
rules,
project_tree,
self._native,
include_trace_on_error=include_trace_on_error)
def execute_request(self, scheduler, product, *subjects):
"""Creates, runs, and returns an ExecutionRequest for the given product and subjects."""
request = scheduler.execution_request([product], subjects)
res = scheduler.execute(request)
if res.error:
raise res.error
return request
def execute(self, scheduler, product, *subjects):
"""Runs an ExecutionRequest for the given product and subjects, and returns the result value."""
request = self.execute_request(scheduler, product, *subjects)
states = [state for _, state in scheduler.root_entries(request)]
if any(type(state) is not Return for state in states):
with temporary_file_path(cleanup=False, suffix='.dot') as dot_file:
scheduler.visualize_graph_to_file(dot_file)
raise ValueError(
'At least one request failed: {}. Visualized as {}'.format(
states, dot_file))
return list(state.value for state in states)
def create_native_scheduler(self, root_subject_types, rules):
rule_index = RuleIndex.create(rules)
native = init_native()
scheduler = WrappedNativeScheduler(native, '.', './.pants.d', [],
rule_index, root_subject_types)
return scheduler
class LegacyAddressMapperTest(unittest.TestCase):
_native = init_native()
def create_build_files(self, build_root):
# Create BUILD files
# build_root:
# BUILD
# BUILD.other
# dir_a:
# BUILD
# BUILD.other
# subdir:
# BUILD
# dir_b:
# BUILD
dir_a = os.path.join(build_root, 'dir_a')
dir_b = os.path.join(build_root, 'dir_b')
dir_a_subdir = os.path.join(dir_a, 'subdir')
safe_mkdir(dir_a)
safe_mkdir(dir_b)
safe_mkdir(dir_a_subdir)
safe_file_dump(os.path.join(build_root, 'BUILD'),
'target(name="a")\ntarget(name="b")')
safe_file_dump(os.path.join(build_root, 'BUILD.other'),
'target(name="c")')
safe_file_dump(os.path.join(dir_a, 'BUILD'),
'target(name="a")\ntarget(name="b")')
safe_file_dump(os.path.join(dir_a, 'BUILD.other'), 'target(name="c")')
safe_file_dump(os.path.join(dir_b, 'BUILD'), 'target(name="a")')
safe_file_dump(os.path.join(dir_a_subdir, 'BUILD'), 'target(name="a")')
def create_address_mapper(self, build_root):
work_dir = os.path.join(build_root, '.pants.d')
scheduler = EngineInitializer.setup_legacy_graph(
[],
work_dir,
build_file_imports_behavior='allow',
build_root=build_root,
native=self._native).scheduler
return LegacyAddressMapper(scheduler.new_session(), build_root)
def test_is_valid_single_address(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
self.assertFalse(
mapper.is_valid_single_address(SingleAddress('dir_a', 'foo')))
self.assertTrue(
mapper.is_valid_single_address(SingleAddress('dir_a', 'a')))
with self.assertRaises(TypeError):
mapper.is_valid_single_address('foo')
def test_scan_build_files(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
build_files = mapper.scan_build_files('')
self.assertEqual(
build_files, {
'BUILD', 'BUILD.other', 'dir_a/BUILD', 'dir_a/BUILD.other',
'dir_b/BUILD', 'dir_a/subdir/BUILD'
})
build_files = mapper.scan_build_files('dir_a/subdir')
self.assertEqual(build_files, {'dir_a/subdir/BUILD'})
def test_scan_build_files_edge_cases(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
# A non-existent dir.
build_files = mapper.scan_build_files('foo')
self.assertEqual(build_files, set())
# A dir with no BUILD files.
safe_mkdir(os.path.join(build_root, 'empty'))
build_files = mapper.scan_build_files('empty')
self.assertEqual(build_files, set())
def test_is_declaring_file(self):
scheduler = mock.Mock()
mapper = LegacyAddressMapper(scheduler, '')
self.assertTrue(
mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD'))
self.assertTrue(
mapper.is_declaring_file(Address('path', 'name'),
'path/BUILD.suffix'))
self.assertFalse(
mapper.is_declaring_file(Address('path', 'name'),
'path/not_a_build_file'))
self.assertFalse(
mapper.is_declaring_file(Address('path', 'name'),
'differing-path/BUILD'))
self.assertFalse(
mapper.is_declaring_file(
BuildFileAddress(target_name='name',
rel_path='path/BUILD.new'), 'path/BUILD'))
self.assertTrue(
mapper.is_declaring_file(
BuildFileAddress(target_name='name', rel_path='path/BUILD'),
'path/BUILD'))
def test_addresses_in_spec_path(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
addresses = mapper.addresses_in_spec_path('dir_a')
self.assertEqual(
addresses, {
Address('dir_a', 'a'),
Address('dir_a', 'b'),
Address('dir_a', 'c')
})
def test_addresses_in_spec_path_no_dir(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
with self.assertRaises(AddressMapper.BuildFileScanError) as cm:
mapper.addresses_in_spec_path('foo')
self.assertIn('does not match any targets.', str(cm.exception))
def test_addresses_in_spec_path_no_build_files(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
safe_mkdir(os.path.join(build_root, 'foo'))
mapper = self.create_address_mapper(build_root)
with self.assertRaises(AddressMapper.BuildFileScanError) as cm:
mapper.addresses_in_spec_path('foo')
self.assertIn('does not match any targets.', str(cm.exception))
def test_scan_specs(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
addresses = mapper.scan_specs(
[SingleAddress('dir_a', 'a'),
SiblingAddresses('')])
self.assertEqual(
addresses, {
Address('', 'a'),
Address('', 'b'),
Address('', 'c'),
Address('dir_a', 'a')
})
def test_scan_specs_bad_spec(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
with self.assertRaises(AddressMapper.BuildFileScanError) as cm:
mapper.scan_specs([SingleAddress('dir_a', 'd')])
self.assertIn('does not match any targets.', str(cm.exception))
def test_scan_addresses(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
addresses = mapper.scan_addresses()
self.assertEqual(
addresses, {
Address('', 'a'),
Address('', 'b'),
Address('', 'c'),
Address('dir_a', 'a'),
Address('dir_a', 'b'),
Address('dir_a', 'c'),
Address('dir_b', 'a'),
Address('dir_a/subdir', 'a')
})
def test_scan_addresses_with_root_specified(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
addresses = mapper.scan_addresses(os.path.join(
build_root, 'dir_a'))
self.assertEqual(
addresses, {
Address('dir_a', 'a'),
Address('dir_a', 'b'),
Address('dir_a', 'c'),
Address('dir_a/subdir', 'a')
})
def test_scan_addresses_bad_dir(self):
# scan_addresses() should not raise an error.
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
addresses = mapper.scan_addresses(os.path.join(build_root, 'foo'))
self.assertEqual(addresses, set())
def test_other_throw_is_fail(self):
# scan_addresses() should raise an error if the scheduler returns an error it can't ignore.
class ThrowReturningScheduler(object):
def execution_request(self, *args):
pass
def execute(self, *args):
return ExecutionResult(
None, [(('some-thing', None),
Throw(Exception('just an exception')))])
with temporary_dir() as build_root:
mapper = LegacyAddressMapper(ThrowReturningScheduler(), build_root)
with self.assertRaises(
LegacyAddressMapper.BuildFileScanError) as cm:
mapper.scan_addresses(os.path.join(build_root, 'foo'))
self.assertIn('just an exception', str(cm.exception))
class LegacyAddressMapperTest(unittest.TestCase):
_native = init_native()
def create_build_files(self, build_root):
# Create BUILD files
# build_root:
# BUILD
# BUILD.other
# dir_a:
# BUILD
# BUILD.other
# subdir:
# BUILD
# dir_b:
# BUILD
dir_a = os.path.join(build_root, 'dir_a')
dir_b = os.path.join(build_root, 'dir_b')
dir_a_subdir = os.path.join(dir_a, 'subdir')
safe_mkdir(dir_a)
safe_mkdir(dir_b)
safe_mkdir(dir_a_subdir)
safe_file_dump(os.path.join(build_root, 'BUILD'),
'target(name="a")\ntarget(name="b")')
safe_file_dump(os.path.join(build_root, 'BUILD.other'),
'target(name="c")')
safe_file_dump(os.path.join(dir_a, 'BUILD'),
'target(name="a")\ntarget(name="b")')
safe_file_dump(os.path.join(dir_a, 'BUILD.other'), 'target(name="c")')
safe_file_dump(os.path.join(dir_b, 'BUILD'), 'target(name="a")')
safe_file_dump(os.path.join(dir_a_subdir, 'BUILD'), 'target(name="a")')
def create_address_mapper(self, build_root):
scheduler, engine, _, _ = EngineInitializer.setup_legacy_graph(
[], build_root=build_root, native=self._native)
return LegacyAddressMapper(scheduler, engine, build_root)
def test_is_valid_single_address(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
self.assertFalse(
mapper.is_valid_single_address(SingleAddress('dir_a', 'foo')))
self.assertTrue(
mapper.is_valid_single_address(SingleAddress('dir_a', 'a')))
with self.assertRaises(TypeError):
mapper.is_valid_single_address('foo')
def test_scan_build_files(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
build_files = mapper.scan_build_files('')
self.assertEqual(
build_files, {
'BUILD', 'BUILD.other', 'dir_a/BUILD', 'dir_a/BUILD.other',
'dir_b/BUILD', 'dir_a/subdir/BUILD'
})
build_files = mapper.scan_build_files('dir_a/subdir')
self.assertEqual(build_files, {'dir_a/subdir/BUILD'})
def test_scan_build_files_edge_cases(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
# A non-existent dir.
build_files = mapper.scan_build_files('foo')
self.assertEqual(build_files, set())
# A dir with no BUILD files.
safe_mkdir(os.path.join(build_root, 'empty'))
build_files = mapper.scan_build_files('empty')
self.assertEqual(build_files, set())
def test_is_declaring_file(self):
scheduler = mock.Mock()
mapper = LegacyAddressMapper(scheduler, None, '')
self.assertTrue(
mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD'))
self.assertTrue(
mapper.is_declaring_file(Address('path', 'name'),
'path/BUILD.suffix'))
self.assertFalse(
mapper.is_declaring_file(Address('path', 'name'),
'path/not_a_build_file'))
self.assertFalse(
mapper.is_declaring_file(Address('path', 'name'),
'differing-path/BUILD'))
def test_addresses_in_spec_path(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
addresses = mapper.addresses_in_spec_path('dir_a')
self.assertEqual(
addresses, {
Address('dir_a', 'a'),
Address('dir_a', 'b'),
Address('dir_a', 'c')
})
def test_addresses_in_spec_path_no_dir(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
with self.assertRaises(AddressMapper.BuildFileScanError):
mapper.addresses_in_spec_path('foo')
# TODO: https://github.com/pantsbuild/pants/issues/4025
# self.assertIn('Directory "foo" does not exist.', str(cm.exception))
def test_addresses_in_spec_path_no_build_files(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
safe_mkdir(os.path.join(build_root, 'foo'))
mapper = self.create_address_mapper(build_root)
with self.assertRaises(AddressMapper.BuildFileScanError):
mapper.addresses_in_spec_path('foo')
# TODO: https://github.com/pantsbuild/pants/issues/4025
# self.assertIn('does not contain build files.', str(cm.exception))
def test_scan_specs(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
addresses = mapper.scan_specs(
[SingleAddress('dir_a', 'a'),
SiblingAddresses('')])
self.assertEqual(
addresses, {
Address('', 'a'),
Address('', 'b'),
Address('', 'c'),
Address('dir_a', 'a')
})
def test_scan_specs_bad_spec(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
with self.assertRaises(AddressMapper.BuildFileScanError):
mapper.scan_specs([SingleAddress('dir_a', 'd')])
# TODO: https://github.com/pantsbuild/pants/issues/4025
# self.assertIn('not found in namespace dir_a for name "d".', str(cm.exception))
def test_scan_addresses(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
addresses = mapper.scan_addresses()
self.assertEqual(
addresses, {
Address('', 'a'),
Address('', 'b'),
Address('', 'c'),
Address('dir_a', 'a'),
Address('dir_a', 'b'),
Address('dir_a', 'c'),
Address('dir_b', 'a'),
Address('dir_a/subdir', 'a')
})
def test_scan_addresses_with_root_specified(self):
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
addresses = mapper.scan_addresses(os.path.join(
build_root, 'dir_a'))
self.assertEqual(
addresses, {
Address('dir_a', 'a'),
Address('dir_a', 'b'),
Address('dir_a', 'c'),
Address('dir_a/subdir', 'a')
})
def test_scan_addresses_bad_dir(self):
# scan_addresses() should not raise an error.
with temporary_dir() as build_root:
self.create_build_files(build_root)
mapper = self.create_address_mapper(build_root)
addresses = mapper.scan_addresses(os.path.join(build_root, 'foo'))
self.assertEqual(addresses, set())
class SchedulerTestBase:
"""A mixin for classes (tests, presumably) which need to create temporary schedulers.
TODO: In the medium term, this should be part of pants_test.test_base.TestBase.
"""
_native = init_native()
def _create_work_dir(self):
work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, work_dir)
return work_dir
def mk_fs_tree(self,
build_root_src=None,
ignore_patterns=None,
work_dir=None):
"""Create a temporary FilesystemProjectTree.
:param build_root_src: Optional directory to pre-populate from; otherwise, empty.
:returns: A FilesystemProjectTree.
"""
work_dir = work_dir or self._create_work_dir()
build_root = os.path.join(work_dir, 'build_root')
if build_root_src is not None:
shutil.copytree(build_root_src, build_root, symlinks=True)
else:
os.makedirs(build_root)
return FileSystemProjectTree(build_root,
ignore_patterns=ignore_patterns)
def mk_scheduler(self,
rules=None,
union_rules=None,
project_tree=None,
work_dir=None,
include_trace_on_error=True):
"""Creates a SchedulerSession for a Scheduler with the given Rules installed."""
rules = rules or []
work_dir = work_dir or self._create_work_dir()
project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir)
local_store_dir = os.path.realpath(safe_mkdtemp())
scheduler = Scheduler(self._native,
project_tree,
local_store_dir,
rules,
union_rules,
DEFAULT_EXECUTION_OPTIONS,
include_trace_on_error=include_trace_on_error)
return scheduler.new_session(zipkin_trace_v2=False,
build_id="buildid_for_test")
def context_with_scheduler(self, scheduler, *args, **kwargs):
return self.context(*args, scheduler=scheduler, **kwargs)
def execute(self, scheduler, product, *subjects):
"""Runs an ExecutionRequest for the given product and subjects, and returns the result value."""
request = scheduler.execution_request([product], subjects)
return self.execute_literal(scheduler, request)
def execute_literal(self, scheduler, execution_request):
returns, throws = scheduler.execute(execution_request)
if throws:
with temporary_file_path(cleanup=False, suffix='.dot') as dot_file:
scheduler.visualize_graph_to_file(dot_file)
raise ValueError(
'At least one root failed: {}. Visualized as {}'.format(
throws, dot_file))
return list(state.value for _, state in returns)
def execute_expecting_one_result(self, scheduler, product, subject):
request = scheduler.execution_request([product], [subject])
returns, throws = scheduler.execute(request)
if throws:
_, state = throws[0]
raise state.exc
self.assertEqual(len(returns), 1)
_, state = returns[0]
return state
def execute_raising_throw(self, scheduler, product, subject):
resulting_value = self.execute_expecting_one_result(
scheduler, product, subject)
self.assertTrue(type(resulting_value) is Throw)
raise resulting_value.exc
class GraphInvalidationTest(unittest.TestCase):
_native = init_native()
def _make_setup_args(self, specs):
options = mock.Mock()
options.target_specs = specs
return options
@contextmanager
def open_scheduler(self, specs, symbol_table_cls=None):
path_ignore_patterns = ['.*']
target_roots = TargetRoots.create(options=self._make_setup_args(specs))
graph_helper = EngineInitializer.setup_legacy_graph(
path_ignore_patterns,
symbol_table_cls=symbol_table_cls,
native=self._native)
graph = graph_helper.create_build_graph(target_roots)[0]
addresses = tuple(graph.inject_specs_closure(target_roots.as_specs()))
yield graph, addresses, graph_helper.scheduler
def test_invalidate_fsnode(self):
with self.open_scheduler(['3rdparty/python::']) as (_, _, scheduler):
initial_node_count = scheduler.node_count()
self.assertGreater(initial_node_count, 0)
invalidated_count = scheduler.invalidate_files(
['3rdparty/python/BUILD'])
self.assertGreater(invalidated_count, 0)
self.assertLess(scheduler.node_count(), initial_node_count)
def test_invalidate_fsnode_incremental(self):
with self.open_scheduler(['//:', '3rdparty/::']) as (_, _, scheduler):
node_count = scheduler.node_count()
self.assertGreater(node_count, 0)
# Invalidate the '3rdparty/python' DirectoryListing, the `3rdparty` DirectoryListing,
# and then the root DirectoryListing by "touching" files/dirs.
# NB: Invalidation of entries in the root directory is special: because Watchman will
# never trigger an event for the root itself, we treat changes to files in the root
# directory as events for the root.
for filename in ('3rdparty/python/BUILD', '3rdparty/python',
'non_existing_file'):
invalidated_count = scheduler.invalidate_files([filename])
self.assertGreater(
invalidated_count, 0,
'File {} did not invalidate any Nodes.'.format(filename))
node_count, last_node_count = scheduler.node_count(
), node_count
self.assertLess(node_count, last_node_count)
def test_sources_ordering(self):
spec = 'testprojects/src/resources/org/pantsbuild/testproject/ordering'
with self.open_scheduler([spec]) as (graph, _, _):
target = graph.get_target(Address.parse(spec))
sources = [
os.path.basename(s)
for s in target.sources_relative_to_buildroot()
]
self.assertEquals(
['p', 'a', 'n', 't', 's', 'b', 'u', 'i', 'l', 'd'], sources)
def test_implicit_sources(self):
expected_sources = {
'testprojects/tests/python/pants/file_sets:implicit_sources':
['a.py', 'aa.py', 'aaa.py', 'aabb.py', 'ab.py'],
'testprojects/tests/python/pants/file_sets:test_with_implicit_sources':
['test_a.py']
}
for spec, exp_sources in expected_sources.items():
with self.open_scheduler([spec]) as (graph, _, _):
target = graph.get_target(Address.parse(spec))
sources = sorted([
os.path.basename(s)
for s in target.sources_relative_to_buildroot()
])
self.assertEquals(exp_sources, sources)
def test_target_macro_override(self):
"""Tests that we can "wrap" an existing target type with additional functionality.
Installs an additional TargetMacro that wraps `target` aliases to add a tag to all definitions.
"""
spec = 'testprojects/tests/python/pants/build_parsing:'
# Confirm that python_tests in a small directory are marked.
with self.open_scheduler(
[spec],
symbol_table_cls=TaggingSymbolTable) as (graph, addresses, _):
self.assertTrue(
len(addresses) > 0,
'No targets matched by {}'.format(addresses))
for address in addresses:
self.assertIn(TaggingSymbolTable.tag,
graph.get_target(address).tags)
