def test_nodes_symlink_globbed_file(self):
self.assert_fsnodes(
['d.ln/b/*.txt'],
[
# NB: Needs to scandir every Dir on the way down to track whether
# it is traversing a symlink.
(Dir(''), DirectoryListing),
# Traverse one symlink.
(Link('d.ln'), ReadLink),
(Dir('a'), DirectoryListing),
(Dir('a/b'), DirectoryListing),
])
def test_nodes_symlink_globbed_dir(self):
self.assert_fsnodes(
['*/2'],
[
# Scandir for the root.
(Dir(''), DirectoryListing),
# Read links to determine whether they're actually directories.
(Link('c.ln'), ReadLink),
(Link('d.ln'), ReadLink),
# Scan second level destinations: `a/b` is matched via `c.ln`.
(Dir('a'), DirectoryListing),
(Dir('a/b'), DirectoryListing),
])
def test_empty(self) -> None:
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(
parser=JsonParser(TEST_TABLE),
prelude_glob_patterns=(),
build_file_imports_behavior=BuildFileImportsBehavior.error,
)
af = run_rule(
parse_address_family,
rule_args=[
address_mapper,
BuildFilePreludeSymbols(FrozenDict()),
Dir("/dev/null")
],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(Digest("abc", 10),
("/dev/null/BUILD", ), ()),
),
MockGet(
product_type=FilesContent,
subject_type=Digest,
mock=lambda _: FilesContent(
[FileContent(path="/dev/null/BUILD", content=b"")]),
),
],
)
self.assertEqual(len(af.objects_by_name), 0)
def test_empty(self):
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(JsonParser(TestTable()))
af = run_rule(parse_address_family, address_mapper, Dir('/dev/null'), {
(FilesContent, PathGlobs): lambda _: FilesContent([FileContent('/dev/null/BUILD', '')])
})
self.assertEquals(len(af.objects_by_name), 0)
class StorageTest(unittest.TestCase):
TEST_KEY = b'hello'
TEST_VALUE = b'world'
TEST_PATH = File('/foo')
TEST_PATH2 = Dir('/bar')
class SomeException(Exception): pass
def setUp(self):
self.storage = Storage.create()
self.result = 'something'
self.request = Runnable(func=_runnable, args=('this is an arg',), cacheable=True)
def test_storage(self):
key = self.storage.put(self.TEST_PATH)
self.assertEquals(self.TEST_PATH, self.storage.get(key))
with self.assertRaises(InvalidKeyError):
self.assertFalse(self.storage.get(self.TEST_KEY))
def test_storage_key_mappings(self):
key1 = self.storage.put(self.TEST_PATH)
key2 = self.storage.put(self.TEST_PATH2)
self.storage.add_mapping(key1, key2)
self.assertEquals(key2, self.storage.get_mapping(key1))
# key2 isn't mapped to any other key.
self.assertIsNone(self.storage.get_mapping(key2))
def generate_subjects(self, filenames):
"""Given filenames, generate a set of subjects for invalidation predicate matching."""
for f in filenames:
# ReadLink, or FileContent for the literal path.
yield File(f)
yield Link(f)
# DirectoryListing for parent dirs.
yield Dir(dirname(f))
def filter_build_dirs(address_mapper, snapshot):
"""Given a Snapshot matching a build pattern, return parent directories as BuildDirs."""
dirnames = set(dirname(f.stat.path) for f in snapshot.files)
ignored_dirnames = address_mapper.build_ignore_patterns.match_files(
'{}/'.format(dirname) for dirname in dirnames)
ignored_dirnames = set(d.rstrip('/') for d in ignored_dirnames)
return BuildDirs(
tuple(Dir(d) for d in dirnames if d not in ignored_dirnames))
def filter_build_dirs(address_mapper, build_files):
"""Given Files matching a build pattern, return their parent directories as BuildDirs."""
dirnames = set(dirname(f.stat.path) for f in build_files.dependencies)
ignored_dirnames = address_mapper.build_ignore_patterns.match_files(
'{}/'.format(dirname) for dirname in dirnames)
ignored_dirnames = set(d.rstrip('/') for d in ignored_dirnames)
return BuildDirs(
tuple(Dir(d) for d in dirnames if d not in ignored_dirnames))
def test_empty(self):
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(JsonParser(TestTable()))
af = run_rule(parse_address_family, address_mapper, Dir('/dev/null'), {
(Snapshot, PathGlobs): lambda _: Snapshot(DirectoryDigest('abc', 10), (File('/dev/null/BUILD'),)),
(FilesContent, DirectoryDigest): lambda _: FilesContent([FileContent('/dev/null/BUILD', b'')]),
})
self.assertEqual(len(af.objects_by_name), 0)
async def find_build_file(address: Address) -> BuildFileAddress:
address_family = await Get[AddressFamily](Dir(address.spec_path))
if address not in address_family.addressables:
_raise_did_you_mean(address_family=address_family,
name=address.target_name)
return next(build_file_address
for build_file_address in address_family.addressables.keys()
if build_file_address == address)
async def provenanced_addresses_from_address_families(
address_mapper: AddressMapper,
address_specs: AddressSpecs,
) -> ProvenancedBuildFileAddresses:
"""Given an AddressMapper and list of AddressSpecs, return matching ProvenancedBuildFileAddresses.
:raises: :class:`ResolveError` if:
- there were no matching AddressFamilies, or
- the AddressSpec matches no addresses for SingleAddresses.
:raises: :class:`AddressLookupError` if no targets are matched for non-SingleAddress specs.
"""
# Capture a Snapshot covering all paths for these AddressSpecs, then group by directory.
snapshot = await Get[Snapshot](PathGlobs,
_address_spec_to_globs(
address_mapper, address_specs))
dirnames = {dirname(f) for f in snapshot.files}
address_families = await MultiGet(Get[AddressFamily](Dir(d))
for d in dirnames)
address_family_by_directory = {af.namespace: af for af in address_families}
matched_addresses = OrderedSet()
addr_to_provenance: Dict[BuildFileAddress, AddressSpec] = {}
for address_spec in address_specs:
# NB: if an address spec is provided which expands to some number of targets, but those targets
# match --exclude-target-regexp, we do NOT fail! This is why we wait to apply the tag and
# exclude patterns until we gather all the targets the address spec would have matched
# without them.
try:
addr_families_for_spec = address_spec.matching_address_families(
address_family_by_directory)
except AddressSpec.AddressFamilyResolutionError as e:
raise ResolveError(e) from e
try:
all_addr_tgt_pairs = address_spec.address_target_pairs_from_address_families(
addr_families_for_spec)
for addr, _ in all_addr_tgt_pairs:
# A target might be covered by multiple specs, so we take the most specific one.
addr_to_provenance[addr] = more_specific(
addr_to_provenance.get(addr), address_spec)
except AddressSpec.AddressResolutionError as e:
raise AddressLookupError(e) from e
except SingleAddress._SingleAddressResolutionError as e:
_raise_did_you_mean(e.single_address_family, e.name, source=e)
matched_addresses.update(
addr for (addr, tgt) in all_addr_tgt_pairs
if address_specs.matcher.matches_target_address_pair(addr, tgt))
# NB: This may be empty, as the result of filtering by tag and exclude patterns!
return ProvenancedBuildFileAddresses(
tuple(
ProvenancedBuildFileAddress(build_file_address=addr,
provenance=addr_to_provenance[addr])
for addr in matched_addresses))
class PathRoot(datatype('PathRoot', []), PathGlob):
"""A PathGlob matching the root of the ProjectTree.
The root is special because it's the only symbolic path that we can implicit trust is
not a symlink.
"""
canonical_stat = Dir('')
symbolic_path = ''
paths = Paths((Path(symbolic_path, canonical_stat), ))
async def find_target_adaptor(address: Address) -> TargetAdaptor:
"""Hydrate a TargetAdaptor so that it may be converted into the Target API."""
if not address.is_base_target:
raise ValueError(
f"Subtargets are not resident in BUILD files, and so do not have TargetAdaptors: {address}"
)
address_family = await Get(AddressFamily, Dir(address.spec_path))
target_adaptor = address_family.get_target_adaptor(address)
if target_adaptor is None:
raise _did_you_mean_exception(address_family, address.target_name)
return target_adaptor
async def find_build_file(address: Address) -> BuildFileAddress:
address_family = await Get(AddressFamily, Dir(address.spec_path))
owning_address = address.maybe_convert_to_base_target()
if address_family.get_target_adaptor(owning_address) is None:
raise _did_you_mean_exception(address_family=address_family,
name=owning_address.target_name)
bfa = next(build_file_address
for build_file_address in address_family.build_file_addresses
if build_file_address.address == owning_address)
return (bfa if address.is_base_target else BuildFileAddress(
rel_path=bfa.rel_path, address=address))
def _lstat(self, relpath):
mode = type(self._reader.lstat(self._scm_relpath(relpath)))
if mode == NoneType:
return None
elif mode == self._reader.Symlink:
return Link(relpath)
elif mode == self._reader.Dir:
return Dir(relpath)
elif mode == self._reader.File:
return File(relpath)
else:
raise IOError('Unsupported file type in {}: {}'.format(self, relpath))
def create_from_spec(cls, canonical_stat, symbolic_path, filespec):
"""Given a filespec, return a tuple of PathGlob objects.
:param canonical_stat: A canonical Dir relative to the ProjectTree, to which the filespec
is relative.
:param symbolic_path: A symbolic name for the canonical_stat (or the same name, if no symlinks
were traversed while expanding it).
:param filespec: A filespec, relative to the canonical_stat.
"""
if not isinstance(canonical_stat, Dir):
raise ValueError(
'Expected a Dir as the canonical_stat. Got: {}'.format(
canonical_stat))
parts = normpath(filespec).split(os_sep)
if canonical_stat == Dir('') and len(parts) == 1 and parts[0] == '.':
# A request for the root path.
return (PathRoot(), )
elif cls._DOUBLE == parts[0]:
parts = cls._prune_doublestar(parts)
if len(parts) == 1:
# Per https://git-scm.com/docs/gitignore:
#
# "A trailing '/**' matches everything inside. For example, 'abc/**' matches all files inside
# directory "abc", relative to the location of the .gitignore file, with infinite depth."
#
return (PathDirWildcard(canonical_stat, symbolic_path, '*',
'**'),
PathWildcard(canonical_stat, symbolic_path, '*'))
# There is a double-wildcard in a dirname of the path: double wildcards are recursive,
# so there are two remainder possibilities: one with the double wildcard included, and the
# other without.
pathglob_with_doublestar = PathDirWildcard(canonical_stat,
symbolic_path, '*',
join(*parts[0:]))
if len(parts) == 2:
pathglob_no_doublestar = PathWildcard(canonical_stat,
symbolic_path, parts[1])
else:
pathglob_no_doublestar = PathDirWildcard(
canonical_stat, symbolic_path, parts[1], join(*parts[2:]))
return (pathglob_with_doublestar, pathglob_no_doublestar)
elif len(parts) == 1:
# This is the path basename.
return (PathWildcard(canonical_stat, symbolic_path, parts[0]), )
else:
# This is a path dirname.
return (PathDirWildcard(canonical_stat, symbolic_path, parts[0],
join(*parts[1:])), )
async def addresses_with_origins_from_address_specs(
address_mapper: AddressMapper,
address_specs: AddressSpecs) -> AddressesWithOrigins:
"""Given an AddressMapper and list of AddressSpecs, return matching AddressesWithOrigins.
:raises: :class:`ResolveError` if there were no matching AddressFamilies or no targets
were matched.
"""
# Snapshot all BUILD files covered by the AddressSpecs, then group by directory.
snapshot = await Get(
Snapshot,
PathGlobs,
address_specs.to_path_globs(
build_patterns=address_mapper.build_patterns,
build_ignore_patterns=address_mapper.build_ignore_patterns,
),
)
dirnames = {os.path.dirname(f) for f in snapshot.files}
address_families = await MultiGet(
Get(AddressFamily, Dir(d)) for d in dirnames)
address_family_by_directory = {af.namespace: af for af in address_families}
matched_addresses: OrderedSet[Address] = OrderedSet()
addr_to_origin: Dict[Address, AddressSpec] = {}
for address_spec in address_specs:
# These may raise ResolveError, depending on the type of spec.
addr_families_for_spec = address_spec.matching_address_families(
address_family_by_directory)
addr_target_pairs_for_spec = address_spec.matching_addresses(
addr_families_for_spec)
if isinstance(address_spec,
SingleAddress) and not addr_target_pairs_for_spec:
addr_family = assert_single_element(addr_families_for_spec)
raise _did_you_mean_exception(addr_family, address_spec.name)
for addr, _ in addr_target_pairs_for_spec:
# A target might be covered by multiple specs, so we take the most specific one.
addr_to_origin[addr] = AddressSpecs.more_specific(
addr_to_origin.get(addr), address_spec)
matched_addresses.update(
addr for (addr, tgt) in addr_target_pairs_for_spec
if (address_specs.filter_by_global_options is False
or address_mapper.matches_filter_options(addr, tgt)))
return AddressesWithOrigins(
AddressWithOrigin(address=addr, origin=addr_to_origin[addr])
for addr in matched_addresses)
class StorageTest(unittest.TestCase):
TEST_KEY = b'hello'
TEST_VALUE = b'world'
TEST_PATH = File('/foo')
TEST_PATH2 = Dir('/bar')
class SomeException(Exception):
pass
def setUp(self):
self.storage = Storage.create()
self.result = 'something'
self.request = Runnable(func=_runnable,
args=('this is an arg', ),
cacheable=True)
def test_lmdb_key_value_store(self):
lmdb = Lmdb.create()[0]
with closing(lmdb) as kvs:
# Initially key does not exist.
self.assertFalse(kvs.get(self.TEST_KEY))
# Now write a key value pair and read back.
written = kvs.put(self.TEST_KEY, self.TEST_VALUE)
self.assertTrue(written)
self.assertEquals(self.TEST_VALUE,
kvs.get(self.TEST_KEY).getvalue())
# Write the same key again will not overwrite.
self.assertFalse(kvs.put(self.TEST_KEY, self.TEST_VALUE))
def test_storage(self):
with closing(self.storage) as storage:
key = storage.put(self.TEST_PATH)
self.assertEquals(self.TEST_PATH, storage.get(key))
with self.assertRaises(InvalidKeyError):
self.assertFalse(storage.get(self.TEST_KEY))
def test_storage_key_mappings(self):
with closing(self.storage) as storage:
key1 = storage.put(self.TEST_PATH)
key2 = storage.put(self.TEST_PATH2)
storage.add_mapping(key1, key2)
self.assertEquals(key2, storage.get_mapping(key1))
# key2 isn't mapped to any other key.
self.assertIsNone(storage.get_mapping(key2))
def resolve_unhydrated_struct(address_mapper, address):
"""Given an AddressMapper and an Address, resolve an UnhydratedStruct.
Recursively collects any embedded addressables within the Struct, but will not walk into a
dependencies field, since those should be requested explicitly by rules.
"""
address_family = yield Get(AddressFamily, Dir(address.spec_path))
struct = address_family.addressables.get(address)
addresses = address_family.addressables
if not struct or address not in addresses:
_raise_did_you_mean(address_family, address.target_name)
dependencies = []
def maybe_append(outer_key, value):
if isinstance(value, six.string_types):
if outer_key != 'dependencies':
dependencies.append(
Address.parse(
value,
relative_to=address.spec_path,
subproject_roots=address_mapper.subproject_roots))
elif isinstance(value, Struct):
collect_dependencies(value)
def collect_dependencies(item):
for key, value in sorted(item._asdict().items(), key=_key_func):
if not AddressableDescriptor.is_addressable(item, key):
continue
if isinstance(value, collections.MutableMapping):
for _, v in sorted(value.items(), key=_key_func):
maybe_append(key, v)
elif isinstance(value, collections.MutableSequence):
for v in value:
maybe_append(key, v)
else:
maybe_append(key, value)
collect_dependencies(struct)
yield UnhydratedStruct(
filter(lambda build_address: build_address == address, addresses)[0],
struct, dependencies)
def test_nodes_symlink_file(self):
self.assert_fsnodes(['c.ln/2'], [
(Dir(''), DirectoryListing),
(Link('c.ln'), ReadLink),
(Dir('a'), DirectoryListing),
(Dir('a/b'), DirectoryListing),
])
self.assert_fsnodes(['d.ln/b/1.txt'], [
(Dir(''), DirectoryListing),
(Link('d.ln'), ReadLink),
(Dir('a'), DirectoryListing),
(Dir('a/b'), DirectoryListing),
])
def addresses_from_address_families(address_mapper: AddressMapper,
specs: Specs) -> BuildFileAddresses:
"""Given an AddressMapper and list of Specs, return matching BuildFileAddresses.
:raises: :class:`ResolveError` if:
- there were no matching AddressFamilies, or
- the Spec matches no addresses for SingleAddresses.
:raises: :class:`AddressLookupError` if no targets are matched for non-SingleAddress specs.
"""
# Capture a Snapshot covering all paths for these Specs, then group by directory.
snapshot = yield Get(Snapshot, PathGlobs,
_spec_to_globs(address_mapper, specs))
dirnames = {dirname(f) for f in snapshot.files}
address_families = yield [Get(AddressFamily, Dir(d)) for d in dirnames]
address_family_by_directory = {af.namespace: af for af in address_families}
matched_addresses = OrderedSet()
for spec in specs:
# NB: if a spec is provided which expands to some number of targets, but those targets match
# --exclude-target-regexp, we do NOT fail! This is why we wait to apply the tag and exclude
# patterns until we gather all the targets the spec would have matched without them.
try:
addr_families_for_spec = spec.matching_address_families(
address_family_by_directory)
except Spec.AddressFamilyResolutionError as e:
raise ResolveError(e) from e
try:
all_addr_tgt_pairs = spec.address_target_pairs_from_address_families(
addr_families_for_spec)
except Spec.AddressResolutionError as e:
raise AddressLookupError(e) from e
except SingleAddress._SingleAddressResolutionError as e:
_raise_did_you_mean(e.single_address_family, e.name, source=e)
matched_addresses.update(
addr for (addr, tgt) in all_addr_tgt_pairs
if specs.matcher.matches_target_address_pair(addr, tgt))
# NB: This may be empty, as the result of filtering by tag and exclude patterns!
yield BuildFileAddresses(tuple(matched_addresses))
def test_parse_address_family_empty() -> None:
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(parser=Parser(
target_type_aliases=[], object_aliases=BuildFileAliases()))
af = run_rule(
parse_address_family,
rule_args=[
address_mapper,
BuildFilePreludeSymbols(FrozenDict()),
Dir("/dev/null")
],
mock_gets=[
MockGet(
product_type=DigestContents,
subject_type=PathGlobs,
mock=lambda _: DigestContents(
[FileContent(path="/dev/null/BUILD", content=b"")]),
),
],
)
assert len(af.name_to_target_adaptors) == 0
def test_empty(self) -> None:
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(JsonParser(TEST_TABLE))
af = run_rule(
parse_address_family,
rule_args=[address_mapper, Dir("/dev/null")],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(Digest("abc", 10),
("/dev/null/BUILD", ), ()),
),
MockGet(
product_type=FilesContent,
subject_type=Digest,
mock=lambda _: FilesContent(
[FileContent(path="/dev/null/BUILD", content=b"")]),
),
],
)
self.assertEqual(len(af.objects_by_name), 0)
def test_parse_address_family_empty() -> None:
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
af = run_rule_with_mocks(
parse_address_family,
rule_args=[
Parser(target_type_aliases=[], object_aliases=BuildFileAliases()),
create_subsystem(GlobalOptions,
build_patterns=["BUILD"],
build_ignore=[]),
BuildFilePreludeSymbols(FrozenDict()),
Dir("/dev/null"),
],
mock_gets=[
MockGet(
product_type=DigestContents,
subject_type=PathGlobs,
mock=lambda _: DigestContents(
[FileContent(path="/dev/null/BUILD", content=b"")]),
),
],
)
assert len(af.name_to_target_adaptors) == 0
def _scandir_raw(self, relpath):
# Sanity check. TODO: this should probably be added to the ProjectTree interface as
# an optional call, so that we can use it in fs.py rather than applying it by default.
abspath = os.path.normpath(self._join(relpath))
if os.path.realpath(abspath) != abspath:
raise ValueError(
'scandir for non-canonical path "{}" not supported in {}.'.
format(relpath, self))
for entry in os.scandir(abspath):
# NB: We don't use `DirEntry.stat`, as the scandir docs indicate that that always requires
# an additional syscall on Unixes.
entry_path = os.path.normpath(os.path.join(relpath, entry.name))
if entry.is_file(follow_symlinks=False):
yield File(entry_path)
elif entry.is_dir(follow_symlinks=False):
yield Dir(entry_path)
elif entry.is_symlink():
yield Link(entry_path)
else:
raise IOError('Unsupported file type in {}: {}'.format(
self, entry_path))
def create_from_spec(cls, canonical_stat, symbolic_path, filespec):
"""Given a filespec, return a PathGlob object.
:param canonical_stat: A canonical Dir relative to the ProjectTree, to which the filespec
is relative.
:param symbolic_path: A symbolic name for the canonical_stat (or the same name, if no symlinks
were traversed while expanding it).
:param filespec: A filespec, relative to the canonical_stat.
"""
if not isinstance(canonical_stat, Dir):
raise ValueError(
'Expected a Dir as the canonical_stat. Got: {}'.format(
canonical_stat))
parts = normpath(filespec).split(os_sep)
if canonical_stat == Dir('') and len(parts) == 1 and parts[0] == '.':
# A request for the root path.
return PathRoot()
elif cls._DOUBLE in parts[0]:
if parts[0] != cls._DOUBLE:
raise ValueError(
'Illegal component "{}" in filespec under {}: {}'.format(
parts[0], symbolic_path, filespec))
# There is a double-wildcard in a dirname of the path: double wildcards are recursive,
# so there are two remainder possibilities: one with the double wildcard included, and the
# other without.
remainders = (join(*parts[1:]), join(*parts[0:]))
return PathDirWildcard(canonical_stat, symbolic_path, parts[0],
remainders)
elif len(parts) == 1:
# This is the path basename.
return PathWildcard(canonical_stat, symbolic_path, parts[0])
else:
# This is a path dirname.
remainders = (join(*parts[1:]), )
return PathDirWildcard(canonical_stat, symbolic_path, parts[0],
remainders)
async def hydrate_struct(address_mapper: AddressMapper,
address: Address) -> HydratedStruct:
"""Given an AddressMapper and an Address, resolve a Struct from a BUILD file.
Recursively collects any embedded addressables within the Struct, but will not walk into a
dependencies field, since those should be requested explicitly by rules.
"""
address_family = await Get[AddressFamily](Dir(address.spec_path))
# NB: `address_family.addressables` is a dictionary of `BuildFileAddress`es and we look it up
# with an `Address`. This works because `BuildFileAddress` is a subclass, but MyPy warns that it
# could be a bug.
struct = address_family.addressables.get(
address) # type: ignore[call-overload]
addresses = address_family.addressables
if not struct or address not in addresses:
_raise_did_you_mean(address_family, address.target_name)
# TODO: This is effectively: "get the BuildFileAddress for this Address".
# see https://github.com/pantsbuild/pants/issues/6657
address = next(build_address for build_address in addresses
if build_address == address)
inline_dependencies = []
def maybe_append(outer_key, value):
if isinstance(value, str):
if outer_key != "dependencies":
inline_dependencies.append(
Address.parse(
value,
relative_to=address.spec_path,
subproject_roots=address_mapper.subproject_roots,
))
elif isinstance(value, Struct):
collect_inline_dependencies(value)
def collect_inline_dependencies(item):
for key, value in sorted(item._asdict().items(), key=_key_func):
if not AddressableDescriptor.is_addressable(item, key):
continue
if isinstance(value, MutableMapping):
for _, v in sorted(value.items(), key=_key_func):
maybe_append(key, v)
elif isinstance(value, MutableSequence):
for v in value:
maybe_append(key, v)
else:
maybe_append(key, value)
# Recursively collect inline dependencies from the fields of the struct into `inline_dependencies`.
collect_inline_dependencies(struct)
# And then hydrate the inline dependencies.
hydrated_inline_dependencies = await MultiGet(
Get[HydratedStruct](Address, a) for a in inline_dependencies)
dependencies = [d.value for d in hydrated_inline_dependencies]
def maybe_consume(outer_key, value):
if isinstance(value, str):
if outer_key == "dependencies":
# Don't recurse into the dependencies field of a Struct, since those will be explicitly
# requested by tasks. But do ensure that their addresses are absolute, since we're
# about to lose the context in which they were declared.
value = Address.parse(
value,
relative_to=address.spec_path,
subproject_roots=address_mapper.subproject_roots,
)
else:
value = dependencies[maybe_consume.idx]
maybe_consume.idx += 1
elif isinstance(value, Struct):
value = consume_dependencies(value)
return value
# NB: Some pythons throw an UnboundLocalError for `idx` if it is a simple local variable.
# TODO(#8496): create a decorator for functions which declare a sentinel variable like this!
maybe_consume.idx = 0 # type: ignore[attr-defined]
# 'zip' the previously-requested dependencies back together as struct fields.
def consume_dependencies(item, args=None):
hydrated_args = args or {}
for key, value in sorted(item._asdict().items(), key=_key_func):
if not AddressableDescriptor.is_addressable(item, key):
hydrated_args[key] = value
continue
if isinstance(value, MutableMapping):
container_type = type(value)
hydrated_args[key] = container_type(
(k, maybe_consume(key, v))
for k, v in sorted(value.items(), key=_key_func))
elif isinstance(value, MutableSequence):
container_type = type(value)
hydrated_args[key] = container_type(
maybe_consume(key, v) for v in value)
else:
hydrated_args[key] = maybe_consume(key, value)
return _hydrate(type(item), address.spec_path, **hydrated_args)
return HydratedStruct(
consume_dependencies(struct, args={"address": address}))
async def hydrate_struct(address_mapper: AddressMapper,
address: Address) -> HydratedStruct:
"""Given an AddressMapper and an Address, resolve a Struct from a BUILD file.
Recursively collects any embedded addressables within the Struct, but will not walk into a
dependencies field, since those should be requested explicitly by rules.
"""
address_family = await Get[AddressFamily](Dir(address.spec_path))
struct = address_family.addressables_as_address_keyed.get(address)
if struct is None:
_raise_did_you_mean(address_family, address.target_name)
inline_dependencies = []
def maybe_append(outer_key, value):
if isinstance(value, str):
if outer_key != "dependencies":
inline_dependencies.append(
Address.parse(
value,
relative_to=address.spec_path,
subproject_roots=address_mapper.subproject_roots,
))
elif isinstance(value, Struct):
collect_inline_dependencies(value)
def collect_inline_dependencies(item):
for key, value in sorted(item._asdict().items(), key=_key_func):
if not AddressableDescriptor.is_addressable(item, key):
continue
if isinstance(value, Mapping):
for _, v in sorted(value.items(), key=_key_func):
maybe_append(key, v)
elif isinstance(value, (list, tuple)):
for v in value:
maybe_append(key, v)
else:
maybe_append(key, value)
# Recursively collect inline dependencies from the fields of the struct into `inline_dependencies`.
collect_inline_dependencies(struct)
# And then hydrate the inline dependencies.
hydrated_inline_dependencies = await MultiGet(
Get[HydratedStruct](Address, a) for a in inline_dependencies)
dependencies = tuple(d.value for d in hydrated_inline_dependencies)
def maybe_consume(outer_key, value):
if isinstance(value, str):
if outer_key == "dependencies":
# Don't recurse into the dependencies field of a Struct, since those will be explicitly
# requested by tasks. But do ensure that their addresses are absolute, since we're
# about to lose the context in which they were declared.
value = Address.parse(
value,
relative_to=address.spec_path,
subproject_roots=address_mapper.subproject_roots,
)
else:
value = dependencies[maybe_consume.idx]
maybe_consume.idx += 1
elif isinstance(value, Struct):
value = consume_dependencies(value)
return value
# NB: Some pythons throw an UnboundLocalError for `idx` if it is a simple local variable.
# TODO(#8496): create a decorator for functions which declare a sentinel variable like this!
maybe_consume.idx = 0 # type: ignore[attr-defined]
# 'zip' the previously-requested dependencies back together as struct fields.
def consume_dependencies(item, args=None):
hydrated_args = args or {}
for key, value in sorted(item._asdict().items(), key=_key_func):
if not AddressableDescriptor.is_addressable(item, key):
hydrated_args[key] = value
continue
if isinstance(value, Mapping):
hydrated_args[key] = {
k: maybe_consume(key, v)
for k, v in sorted(value.items(), key=_key_func)
}
elif isinstance(value, (list, tuple)):
hydrated_args[key] = tuple(
maybe_consume(key, v) for v in value)
else:
hydrated_args[key] = maybe_consume(key, value)
return _hydrate(type(item), address.spec_path, **hydrated_args)
return HydratedStruct(
consume_dependencies(struct, args={"address": address}))
async def addresses_with_origins_from_address_specs(
address_specs: AddressSpecs, global_options: GlobalOptions, specs_filter: AddressSpecsFilter
) -> AddressesWithOrigins:
"""Given an AddressMapper and list of AddressSpecs, return matching AddressesWithOrigins.
:raises: :class:`ResolveError` if the provided specs fail to match targets, and those spec
types expect to have matched something.
"""
matched_addresses: OrderedSet[Address] = OrderedSet()
addr_to_origin: Dict[Address, AddressSpec] = {}
filtering_disabled = address_specs.filter_by_global_options is False
# First convert all `AddressLiteralSpec`s. Some of the resulting addresses may be file
# addresses. This will raise an exception if any of the addresses are not valid.
literal_addresses = await MultiGet(
Get(Address, AddressInput(spec.path_component, spec.target_component))
for spec in address_specs.literals
)
literal_target_adaptors = await MultiGet(
Get(TargetAdaptor, Address, addr.maybe_convert_to_base_target())
for addr in literal_addresses
)
# We convert to targets for the side effect of validating that any file addresses actually
# belong to the specified base targets.
await Get(
UnexpandedTargets, Addresses(addr for addr in literal_addresses if not addr.is_base_target)
)
for literal_spec, addr, target_adaptor in zip(
address_specs.literals, literal_addresses, literal_target_adaptors
):
addr_to_origin[addr] = literal_spec
if filtering_disabled or specs_filter.matches(addr, target_adaptor):
matched_addresses.add(addr)
# Then, convert all `AddressGlobSpecs`. Snapshot all BUILD files covered by the specs, then
# group by directory.
snapshot = await Get(
Snapshot,
PathGlobs,
address_specs.to_path_globs(
build_patterns=global_options.options.build_patterns,
build_ignore_patterns=global_options.options.build_ignore,
),
)
dirnames = {os.path.dirname(f) for f in snapshot.files}
address_families = await MultiGet(Get(AddressFamily, Dir(d)) for d in dirnames)
address_family_by_directory = {af.namespace: af for af in address_families}
for glob_spec in address_specs.globs:
# These may raise ResolveError, depending on the type of spec.
addr_families_for_spec = glob_spec.matching_address_families(address_family_by_directory)
addr_target_pairs_for_spec = glob_spec.matching_addresses(addr_families_for_spec)
for addr, _ in addr_target_pairs_for_spec:
# A target might be covered by multiple specs, so we take the most specific one.
addr_to_origin[addr] = AddressSpecs.more_specific(addr_to_origin.get(addr), glob_spec)
matched_addresses.update(
addr
for (addr, tgt) in addr_target_pairs_for_spec
if filtering_disabled or specs_filter.matches(addr, tgt)
)
return AddressesWithOrigins(
AddressWithOrigin(address=addr, origin=addr_to_origin[addr]) for addr in matched_addresses
)
def test_nodes_file(self):
self.assert_fsnodes(['4.txt'], [
(Dir(''), DirectoryListing),
])
