def test_resolve_without_a_matching_target(self):
graph_mock = mock.Mock()
graph_mock.get_target = mock.Mock(return_value=None)
graph_mock.inject_specs_closure = mock.Mock(return_value=[Address('a','different')])
mapper = LegacyAddressMapper(graph_mock, '')
with self.assertRaises(AddressMapper.BuildFileScanError):
mapper.resolve(Address('a', 'address'))
def test_scan_specs_returns_ordered_set(self):
address = Address('a', 'b')
graph_mock = mock.Mock()
graph_mock.inject_specs_closure = mock.Mock(return_value=[address, address])
mapper = LegacyAddressMapper(graph_mock, '')
self.assertEqual(OrderedSet([address]), mapper.scan_specs([SiblingAddresses('any')]))
def test_addresses_in_spec_path_wraps_error_in_buildfile_scan_error(self):
graph_mock = mock.Mock()
graph_mock.inject_specs_closure = mock.Mock(side_effect=LegacyBuildGraph.InvalidCommandLineSpecError('some msg'))
mapper = LegacyAddressMapper(graph_mock, '')
with self.assertRaises(AddressMapper.BuildFileScanError) as cm:
mapper.addresses_in_spec_path('some/path')
self.assertEqual('some msg', str(cm.exception))
def test_resolve_without_a_matching_target(self):
graph_mock = mock.Mock()
graph_mock.get_target = mock.Mock(return_value=None)
graph_mock.inject_specs_closure = mock.Mock(
return_value=[Address('a', 'different')])
mapper = LegacyAddressMapper(graph_mock, '')
with self.assertRaises(AddressMapper.BuildFileScanError):
mapper.resolve(Address('a', 'address'))
def test_resolve_with_a_target(self):
target = LegacyTarget(None, None)
address = Address('a', 'a')
graph_mock = mock.Mock()
graph_mock.get_target = mock.Mock(return_value=target)
mapper = LegacyAddressMapper(graph_mock, '')
self.assertEqual((address, target), mapper.resolve(address))
def test_resolve_with_a_target(self):
target = LegacyTarget(None, None)
address = Address('a', 'a')
graph_mock = mock.Mock()
graph_mock.get_target = mock.Mock(return_value=target)
mapper = LegacyAddressMapper(graph_mock, '')
self.assertEqual((address, target), mapper.resolve(address))
def test_scan_specs_returns_ordered_set(self):
address = Address('a', 'b')
graph_mock = mock.Mock()
graph_mock.inject_specs_closure = mock.Mock(
return_value=[address, address])
mapper = LegacyAddressMapper(graph_mock, '')
self.assertEqual(OrderedSet([address]),
mapper.scan_specs([SiblingAddresses('any')]))
def test_addresses_in_spec_path_wraps_error_in_buildfile_scan_error(self):
graph_mock = mock.Mock()
graph_mock.inject_specs_closure = mock.Mock(
side_effect=LegacyBuildGraph.InvalidCommandLineSpecError(
'some msg'))
mapper = LegacyAddressMapper(graph_mock, '')
with self.assertRaises(AddressMapper.BuildFileScanError) as cm:
mapper.addresses_in_spec_path('some/path')
self.assertEqual('some msg', str(cm.exception))
def test_scan_addresses_with_root_specified(self):
address = Address('a', 'b')
graph_mock = mock.Mock()
graph_mock.inject_specs_closure = mock.Mock(return_value=[address])
mapper = LegacyAddressMapper(graph_mock, '/some/build/root')
absolute_root_path = '/some/build/root/a'
mapper.scan_addresses(absolute_root_path)
graph_mock.inject_specs_closure.assert_called_with([DescendantAddresses('a')])
def test_scan_addresses_with_root_specified(self):
address = Address('a', 'b')
graph_mock = mock.Mock()
graph_mock.inject_specs_closure = mock.Mock(return_value=[address])
mapper = LegacyAddressMapper(graph_mock, '/some/build/root')
absolute_root_path = '/some/build/root/a'
mapper.scan_addresses(absolute_root_path)
graph_mock.inject_specs_closure.assert_called_with(
[DescendantAddresses('a')])
def test_other_throw_is_fail(self) -> None:
# scan_addresses() should raise an error if the scheduler returns an error it can't ignore.
class ThrowReturningScheduler:
def execution_request(self, *args):
pass
def execute(self, *args):
return [], [(("some-thing", None), Throw(Exception("just an exception")))]
with temporary_dir() as build_root:
mapper = LegacyAddressMapper(ThrowReturningScheduler(), build_root)
with self.assertRaises(LegacyAddressMapper.BuildFileScanError) as cm:
mapper.scan_addresses(os.path.join(build_root, "foo"))
self.assertIn("just an exception", str(cm.exception))
def test_other_throw_is_fail(self):
# scan_addresses() should raise an error if the scheduler returns an error it can't ignore.
class ThrowReturningScheduler(object):
def execution_request(self, *args):
pass
def execute(self, *args):
return [], [(('some-thing', None), Throw(Exception('just an exception')))]
with temporary_dir() as build_root:
mapper = LegacyAddressMapper(ThrowReturningScheduler(), build_root)
with self.assertRaises(LegacyAddressMapper.BuildFileScanError) as cm:
mapper.scan_addresses(os.path.join(build_root, 'foo'))
self.assertIn('just an exception', str(cm.exception))
def test_other_throw_is_fail(self):
# scan_addresses() should raise an error if the scheduler returns an error it can't ignore.
class ThrowReturningScheduler(object):
def execution_request(self, *args):
pass
def execute(self, *args):
return ExecutionResult(None, [(('some-thing', None), Throw(Exception('just an exception')))])
with temporary_dir() as build_root:
mapper = LegacyAddressMapper(ThrowReturningScheduler(), build_root)
with self.assertRaises(LegacyAddressMapper.BuildFileScanError) as cm:
mapper.scan_addresses(os.path.join(build_root, 'foo'))
self.assertIn('just an exception', str(cm.exception))
def create_build_graph(self,
target_roots,
build_root=None,
include_trace_on_error=True):
"""Construct and return a `BuildGraph` given a set of input specs.
:param TargetRoots target_roots: The targets root of the request.
:param string build_root: The build root.
:returns: A tuple of (BuildGraph, AddressMapper).
"""
logger.debug('target_roots are: %r', target_roots)
graph = LegacyBuildGraph.create(
self.scheduler,
self.engine,
self.symbol_table_cls,
include_trace_on_error=include_trace_on_error)
logger.debug('build_graph is: %s', graph)
with self.scheduler.locked():
# Ensure the entire generator is unrolled.
for _ in graph.inject_specs_closure(target_roots.as_specs()):
pass
logger.debug('engine cache stats: %s', self.engine.cache_stats())
address_mapper = LegacyAddressMapper(self.scheduler, self.engine,
build_root or get_buildroot())
logger.debug('address_mapper is: %s', address_mapper)
return graph, address_mapper
async def find_owners(owners_request: OwnersRequest) -> Owners:
sources_set = FrozenOrderedSet(owners_request.sources)
dirs_set = FrozenOrderedSet(
os.path.dirname(source) for source in sources_set)
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set)
candidate_targets = await Get[HydratedTargets](
AddressSpecs(candidate_specs))
# Match the source globs against the expanded candidate targets.
def owns_any_source(legacy_target: HydratedTarget) -> bool:
"""Given a `HydratedTarget` instance, check if it owns the given source file."""
target_kwargs = legacy_target.adaptor.kwargs()
# Handle `sources`-declaring targets.
# NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target,
# so we don't actually call `fileset.matches` here.
# TODO: This matching logic should be implemented using the rust `fs` crate for two reasons:
# 1) having two implementations isn't great
# 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match
target_sources = target_kwargs.get("sources", None)
return target_sources and any_matches_filespec(
paths=sources_set, spec=target_sources.filespec)
build_file_addresses = await MultiGet(
Get[BuildFileAddress](Address, ht.adaptor.address)
for ht in candidate_targets)
owners = Addresses(
ht.adaptor.address
for ht, bfa in zip(candidate_targets, build_file_addresses)
if LegacyAddressMapper.any_is_declaring_file(bfa, sources_set)
or owns_any_source(ht))
return Owners(owners)
def iter_target_addresses_for_sources(self, sources):
"""Bulk, iterable form of `target_addresses_for_source`."""
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
sources_set = set(sources)
subjects = [
AscendantAddresses(directory=d)
for d in self._unique_dirs_for_sources(sources_set)
]
for hydrated_targets in self._engine.product_request(
HydratedTargets, subjects):
for hydrated_target in hydrated_targets.dependencies:
legacy_address = hydrated_target.adaptor.address
# Handle BUILD files.
if any(
LegacyAddressMapper.is_declaring_file(
legacy_address, f) for f in sources_set):
yield legacy_address
else:
# Handle claimed files.
target_files_iter = self._iter_owned_files_from_hydrated_target(
hydrated_target)
if any(source_file in sources_set
for source_file in target_files_iter):
# At least one file in this targets sources match our changed sources - emit its address.
yield legacy_address
def create_address_mapper(self, build_root):
work_dir = os.path.join(build_root, '.pants.d')
scheduler = EngineInitializer.setup_legacy_graph(
[],
work_dir,
build_file_imports_behavior='allow',
build_root=build_root,
native=self._native).scheduler
return LegacyAddressMapper(scheduler.new_session(), build_root)
async def find_owners(build_configuration: BuildConfiguration,
address_mapper: AddressMapper,
owners_request: OwnersRequest) -> BuildFileAddresses:
sources_set = OrderedSet(owners_request.sources)
dirs_set = OrderedSet(dirname(source) for source in sources_set)
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set)
candidate_targets = await Get(HydratedTargets, Specs(candidate_specs))
# Match the source globs against the expanded candidate targets.
def owns_any_source(legacy_target):
"""Given a `HydratedTarget` instance, check if it owns the given source file."""
target_kwargs = legacy_target.adaptor.kwargs()
# Handle `sources`-declaring targets.
# NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target,
# so we don't actually call `fileset.matches` here.
# TODO: This matching logic should be implemented using the rust `fs` crate for two reasons:
# 1) having two implementations isn't great
# 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match
target_sources = target_kwargs.get('sources', None)
if target_sources and any_matches_filespec(sources_set,
target_sources.filespec):
return True
return False
direct_owners = tuple(
ht.adaptor.address for ht in candidate_targets
if LegacyAddressMapper.any_is_declaring_file(
ht.adaptor.address, sources_set) or owns_any_source(ht))
# If the OwnersRequest does not require dependees, then we're done.
if owners_request.include_dependees == 'none':
return BuildFileAddresses(direct_owners)
else:
# Otherwise: find dependees.
all_addresses = await Get(BuildFileAddresses,
Specs((DescendantAddresses(''), )))
all_hydrated_structs = await MultiGet(
Get(HydratedStruct, Address, a.to_address())
for a in all_addresses)
all_structs = [hs.value for hs in all_hydrated_structs]
bfa = build_configuration.registered_aliases()
graph = _DependentGraph.from_iterable(
target_types_from_build_file_aliases(bfa), address_mapper,
all_structs)
if owners_request.include_dependees == 'direct':
return BuildFileAddresses(
tuple(graph.dependents_of_addresses(direct_owners)))
else:
assert owners_request.include_dependees == 'transitive'
return BuildFileAddresses(
tuple(graph.transitive_dependents_of_addresses(direct_owners)))
def test_is_declaring_file(self):
scheduler = mock.Mock()
mapper = LegacyAddressMapper(scheduler, '')
self.assertTrue(mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD'))
self.assertTrue(mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD.suffix'))
self.assertFalse(mapper.is_declaring_file(Address('path', 'name'), 'path/not_a_build_file'))
self.assertFalse(mapper.is_declaring_file(Address('path', 'name'), 'differing-path/BUILD'))
self.assertFalse(mapper.is_declaring_file(
BuildFileAddress(target_name='name', rel_path='path/BUILD.new'),
'path/BUILD'))
self.assertTrue(mapper.is_declaring_file(
BuildFileAddress(target_name='name', rel_path='path/BUILD'),
'path/BUILD'))
def test_is_declaring_file(self):
scheduler = mock.Mock()
mapper = LegacyAddressMapper(scheduler, None, '')
self.assertTrue(mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD'))
self.assertTrue(mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD.suffix'))
self.assertFalse(mapper.is_declaring_file(Address('path', 'name'), 'path/not_a_build_file'))
self.assertFalse(mapper.is_declaring_file(Address('path', 'name'), 'differing-path/BUILD'))
def create_build_graph(
self, specs: Specs, build_root: Optional[str] = None,
) -> Tuple[LegacyBuildGraph, LegacyAddressMapper]:
"""Construct and return a `BuildGraph` given a set of input specs."""
logger.debug("specs are: %r", specs)
graph = LegacyBuildGraph.create(self.scheduler_session, self.build_file_aliases)
logger.debug("build_graph is: %s", graph)
# Ensure the entire generator is unrolled.
for _ in graph.inject_roots_closure(specs.address_specs):
pass
address_mapper = LegacyAddressMapper(self.scheduler_session, build_root or get_buildroot())
logger.debug("address_mapper is: %s", address_mapper)
return graph, address_mapper
def test_is_declaring_file(self):
scheduler = mock.Mock()
mapper = LegacyAddressMapper(scheduler, '')
self.assertTrue(mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD'))
self.assertTrue(mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD.suffix'))
self.assertFalse(mapper.is_declaring_file(Address('path', 'name'), 'path/not_a_build_file'))
self.assertFalse(mapper.is_declaring_file(Address('path', 'name'), 'differing-path/BUILD'))
self.assertFalse(mapper.is_declaring_file(
BuildFileAddress(target_name='name', rel_path='path/BUILD.new'),
'path/BUILD'))
self.assertTrue(mapper.is_declaring_file(
BuildFileAddress(target_name='name', rel_path='path/BUILD'),
'path/BUILD'))
def test_is_declaring_file(self) -> None:
scheduler = unittest.mock.Mock()
mapper = LegacyAddressMapper(scheduler, "")
self.assertTrue(mapper.is_declaring_file(Address("path", "name"), "path/BUILD"))
self.assertTrue(mapper.is_declaring_file(Address("path", "name"), "path/BUILD.suffix"))
self.assertFalse(mapper.is_declaring_file(Address("path", "name"), "path/not_a_build_file"))
self.assertFalse(mapper.is_declaring_file(Address("path", "name"), "differing-path/BUILD"))
self.assertFalse(
mapper.is_declaring_file(
BuildFileAddress(target_name="name", rel_path="path/BUILD.new"), "path/BUILD"
)
)
self.assertTrue(
mapper.is_declaring_file(
BuildFileAddress(target_name="name", rel_path="path/BUILD"), "path/BUILD"
)
)
def find_owners(build_configuration, address_mapper, owners_request):
sources_set = OrderedSet(owners_request.sources)
dirs_set = OrderedSet(dirname(source) for source in sources_set)
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set)
candidate_targets = yield Get(HydratedTargets, Specs(candidate_specs))
# Match the source globs against the expanded candidate targets.
def owns_any_source(legacy_target):
"""Given a `HydratedTarget` instance, check if it owns the given source file."""
target_kwargs = legacy_target.adaptor.kwargs()
# Handle `sources`-declaring targets.
# NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target,
# so we don't actually call `fileset.matches` here.
# TODO: This matching logic should be implemented using the rust `fs` crate for two reasons:
# 1) having two implementations isn't great
# 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match
target_sources = target_kwargs.get('sources', None)
if target_sources and any_matches_filespec(sources_set, target_sources.filespec):
return True
return False
direct_owners = tuple(ht.adaptor.address
for ht in candidate_targets
if LegacyAddressMapper.any_is_declaring_file(ht.adaptor.address, sources_set) or
owns_any_source(ht))
# If the OwnersRequest does not require dependees, then we're done.
if owners_request.include_dependees == 'none':
yield BuildFileAddresses(direct_owners)
else:
# Otherwise: find dependees.
all_addresses = yield Get(BuildFileAddresses, Specs((DescendantAddresses(''),)))
all_structs = yield [Get(HydratedStruct, Address, a.to_address()) for a in all_addresses]
all_structs = [s.value for s in all_structs]
bfa = build_configuration.registered_aliases()
graph = _DependentGraph.from_iterable(target_types_from_build_file_aliases(bfa),
address_mapper,
all_structs)
if owners_request.include_dependees == 'direct':
yield BuildFileAddresses(tuple(graph.dependents_of_addresses(direct_owners)))
else:
assert owners_request.include_dependees == 'transitive'
yield BuildFileAddresses(tuple(graph.transitive_dependents_of_addresses(direct_owners)))
def iter_target_addresses_for_sources(self, sources):
"""Bulk, iterable form of `target_addresses_for_source`."""
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
sources_set = set(sources)
subjects = [AscendantAddresses(directory=d) for d in self._unique_dirs_for_sources(sources_set)]
for hydrated_targets in self._engine.product_request(HydratedTargets, subjects):
for hydrated_target in hydrated_targets.dependencies:
legacy_address = hydrated_target.adaptor.address
# Handle BUILD files.
if any(LegacyAddressMapper.is_declaring_file(legacy_address, f) for f in sources_set):
yield legacy_address
else:
if any(self._owns_source(source, hydrated_target) for source in sources_set):
yield legacy_address
def create_build_graph(self, target_roots, build_root=None):
"""Construct and return a `BuildGraph` given a set of input specs.
:param TargetRoots target_roots: The targets root of the request.
:param string build_root: The build root.
:returns: A tuple of (BuildGraph, AddressMapper).
"""
logger.debug('target_roots are: %r', target_roots)
graph = LegacyBuildGraph.create(self.scheduler_session, self.build_file_aliases)
logger.debug('build_graph is: %s', graph)
# Ensure the entire generator is unrolled.
for _ in graph.inject_roots_closure(target_roots):
pass
address_mapper = LegacyAddressMapper(self.scheduler_session, build_root or get_buildroot())
logger.debug('address_mapper is: %s', address_mapper)
return graph, address_mapper
def iter_target_addresses_for_sources(self, sources):
"""Bulk, iterable form of `target_addresses_for_source`."""
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
sources_set = set(sources)
specs = tuple(AscendantAddresses(directory=d) for d in self._unique_dirs_for_sources(sources_set))
# Uniqify all transitive hydrated targets.
hydrated_target_to_address = {}
hydrated_targets, = self._scheduler.product_request(HydratedTargets, [Specs(specs)])
for hydrated_target in hydrated_targets.dependencies:
if hydrated_target not in hydrated_target_to_address:
hydrated_target_to_address[hydrated_target] = hydrated_target.adaptor.address
for hydrated_target, legacy_address in six.iteritems(hydrated_target_to_address):
# Handle BUILD files.
if (LegacyAddressMapper.any_is_declaring_file(legacy_address, sources_set) or
self._owns_any_source(sources_set, hydrated_target)):
yield legacy_address
def iter_target_addresses_for_sources(self, sources):
"""Bulk, iterable form of `target_addresses_for_source`."""
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
sources_set = set(sources)
subjects = [AscendantAddresses(directory=d) for d in self._unique_dirs_for_sources(sources_set)]
for legacy_target in self._engine.product_request(LegacyTarget, subjects):
legacy_address = legacy_target.adaptor.address
# Handle BUILD files.
if any(LegacyAddressMapper.is_declaring_file(legacy_address, f) for f in sources_set):
yield legacy_address
else:
# Handle claimed files.
target_files_iter = self._iter_owned_files_from_legacy_target(legacy_target)
if any(source_file in sources_set for source_file in target_files_iter):
# At least one file in this targets sources match our changed sources - emit its address.
yield legacy_address
def iter_target_addresses_for_sources(self, sources):
"""Bulk, iterable form of `target_addresses_for_source`."""
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
sources_set = set(sources)
subjects = [
AscendantAddresses(directory=d)
for d in self._unique_dirs_for_sources(sources_set)
]
for hydrated_targets in self._scheduler.product_request(
HydratedTargets, subjects):
for hydrated_target in hydrated_targets.dependencies:
legacy_address = hydrated_target.adaptor.address
# Handle BUILD files.
if any(
LegacyAddressMapper.is_declaring_file(
legacy_address, f) for f in sources_set):
yield legacy_address
else:
if any(
self._owns_source(source, hydrated_target)
for source in sources_set):
yield legacy_address
def create_address_mapper(self, build_root):
work_dir = os.path.join(build_root, '.pants.d')
scheduler, engine, _, _ = EngineInitializer.setup_legacy_graph(
[], work_dir, build_root=build_root, native=self._native)
return LegacyAddressMapper(scheduler, engine, build_root)
def create_address_mapper(self, build_root): scheduler, engine, _, _ = EngineInitializer.setup_legacy_graph([], build_root=build_root, native=self._native) return LegacyAddressMapper(scheduler, engine, build_root)
