def test_address_specs_deduplication(
address_specs_rule_runner: RuleRunner) -> None:
"""When multiple specs cover the same address, we should deduplicate to one single
AddressWithOrigin.
We should use the most specific origin spec possible, such as AddressLiteralSpec >
SiblingAddresses.
"""
address_specs_rule_runner.create_file("demo/f.txt")
address_specs_rule_runner.add_to_build_file("demo",
"mock_tgt(sources=['f.txt'])")
# We also include a file address to ensure that that is included in the result.
specs = [
AddressLiteralSpec("demo", "demo"),
AddressLiteralSpec("demo/f.txt", "demo"),
SiblingAddresses("demo"),
DescendantAddresses("demo"),
AscendantAddresses("demo"),
]
assert resolve_address_specs(address_specs_rule_runner, specs) == {
AddressWithOrigin(Address("demo"), AddressLiteralSpec("demo", "demo")),
AddressWithOrigin(
Address("demo", relative_file_path="f.txt"),
AddressLiteralSpec("demo/f.txt", "demo"),
),
}
async def addresses_with_origins_from_filesystem_specs(
filesystem_specs: FilesystemSpecs, global_options: GlobalOptions,
) -> AddressesWithOrigins:
"""Find the owner(s) for each FilesystemSpec while preserving the original FilesystemSpec those
owners come from.
This will merge FilesystemSpecs that come from the same owning target into a single
FilesystemMergedSpec.
"""
pathglobs_per_include = (
filesystem_specs.path_globs_for_spec(
spec, global_options.options.owners_not_found_behavior.to_glob_match_error_behavior(),
)
for spec in filesystem_specs.includes
)
snapshot_per_include = await MultiGet(
Get[Snapshot](PathGlobs, pg) for pg in pathglobs_per_include
)
owners_per_include = await MultiGet(
Get[Owners](OwnersRequest(sources=snapshot.files)) for snapshot in snapshot_per_include
)
addresses_to_specs: DefaultDict[
Address, List[Union[FilesystemLiteralSpec, FilesystemResolvedGlobSpec]]
] = defaultdict(list)
for spec, snapshot, owners in zip(
filesystem_specs.includes, snapshot_per_include, owners_per_include
):
if (
global_options.options.owners_not_found_behavior != OwnersNotFoundBehavior.ignore
and isinstance(spec, FilesystemLiteralSpec)
and not owners.addresses
):
file_path = PurePath(spec.to_spec_string())
msg = (
f"No owning targets could be found for the file `{file_path}`.\n\nPlease check "
f"that there is a BUILD file in `{file_path.parent}` with a target whose `sources` field "
f"includes `{file_path}`. See https://pants.readme.io/docs/targets for more "
"information on target definitions.\n"
"If you would like to ignore un-owned files, please pass `--owners-not-found-behavior=ignore`."
)
if global_options.options.owners_not_found_behavior == OwnersNotFoundBehavior.warn:
logger.warning(msg)
else:
raise ResolveError(msg)
# We preserve what literal files any globs resolved to. This allows downstream goals to be
# more precise in which files they operate on.
origin: Union[FilesystemLiteralSpec, FilesystemResolvedGlobSpec] = (
spec
if isinstance(spec, FilesystemLiteralSpec)
else FilesystemResolvedGlobSpec(glob=spec.glob, files=snapshot.files)
)
for address in owners.addresses:
addresses_to_specs[address].append(origin)
return AddressesWithOrigins(
AddressWithOrigin(
address, specs[0] if len(specs) == 1 else FilesystemMergedSpec.create(specs)
)
for address, specs in addresses_to_specs.items()
)
def test_strip_address_origin() -> None:
addr = Address.parse("//:demo")
result = run_rule(
strip_address_origins,
rule_args=[
AddressesWithOrigins(
[AddressWithOrigin(addr, SingleAddress("", "demo"))])
],
)
assert list(result) == [addr]
def test_address_specs_filter_by_tag(
address_specs_rule_runner: RuleRunner) -> None:
address_specs_rule_runner.create_file("demo/f.txt")
address_specs_rule_runner.add_to_build_file(
"demo",
dedent("""\
mock_tgt(name="a", sources=["f.txt"])
mock_tgt(name="b", sources=["f.txt"], tags=["integration"])
mock_tgt(name="c", sources=["f.txt"], tags=["ignore"])
"""),
)
bootstrapper = create_options_bootstrapper(args=["--tag=+integration"])
assert resolve_address_specs(address_specs_rule_runner,
[SiblingAddresses("demo")],
bootstrapper=bootstrapper) == {
AddressWithOrigin(
Address("demo", target_name="b"),
SiblingAddresses("demo"))
}
# The same filtering should work when given literal addresses, including file addresses.
# For file addresses, we look up the `tags` field of the original base target.
literals_result = resolve_address_specs(
address_specs_rule_runner,
[
AddressLiteralSpec("demo", "a"),
AddressLiteralSpec("demo", "b"),
AddressLiteralSpec("demo", "c"),
AddressLiteralSpec("demo/f.txt", "a"),
AddressLiteralSpec("demo/f.txt", "b"),
AddressLiteralSpec("demo/f.txt", "c"),
],
bootstrapper=bootstrapper,
)
assert literals_result == {
AddressWithOrigin(
Address("demo", relative_file_path="f.txt", target_name="b"),
AddressLiteralSpec("demo/f.txt", "b"),
),
AddressWithOrigin(Address("demo", target_name="b"),
AddressLiteralSpec("demo", "b")),
}
def test_strip_address_origin() -> None:
addr = Address.parse("//:demo")
result = run_rule_with_mocks(
strip_address_origins,
rule_args=[
AddressesWithOrigins(
[AddressWithOrigin(addr, AddressLiteralSpec("", "demo"))])
],
)
assert list(result) == [addr]
async def addresses_with_origins_from_address_families(
address_mapper: AddressMapper,
address_specs: AddressSpecs,
) -> AddressesWithOrigins:
"""Given an AddressMapper and list of AddressSpecs, return matching AddressesWithOrigins.
:raises: :class:`ResolveError` if:
- there were no matching AddressFamilies, or
- the AddressSpec matches no addresses for SingleAddresses.
:raises: :class:`AddressLookupError` if no targets are matched for non-SingleAddress specs.
"""
# Capture a Snapshot covering all paths for these AddressSpecs, then group by directory.
snapshot = await Get[Snapshot](PathGlobs,
_address_spec_to_globs(
address_mapper, address_specs))
dirnames = {os.path.dirname(f) for f in snapshot.files}
address_families = await MultiGet(Get[AddressFamily](Dir(d))
for d in dirnames)
address_family_by_directory = {af.namespace: af for af in address_families}
matched_addresses: OrderedSet[Address] = OrderedSet()
addr_to_origin: Dict[Address, AddressSpec] = {}
for address_spec in address_specs:
# NB: if an address spec is provided which expands to some number of targets, but those targets
# match --exclude-target-regexp, we do NOT fail! This is why we wait to apply the tag and
# exclude patterns until we gather all the targets the address spec would have matched
# without them.
try:
addr_families_for_spec = address_spec.matching_address_families(
address_family_by_directory)
except AddressSpec.AddressFamilyResolutionError as e:
raise ResolveError(e) from e
try:
all_bfaddr_tgt_pairs = address_spec.address_target_pairs_from_address_families(
addr_families_for_spec)
for bfaddr, _ in all_bfaddr_tgt_pairs:
addr = bfaddr.to_address()
# A target might be covered by multiple specs, so we take the most specific one.
addr_to_origin[addr] = more_specific(addr_to_origin.get(addr),
address_spec)
except AddressSpec.AddressResolutionError as e:
raise AddressLookupError(e) from e
except SingleAddress._SingleAddressResolutionError as e:
_raise_did_you_mean(e.single_address_family, e.name, source=e)
matched_addresses.update(
bfaddr.to_address() for (bfaddr, tgt) in all_bfaddr_tgt_pairs
if address_specs.matcher.matches_target_address_pair(bfaddr, tgt))
# NB: This may be empty, as the result of filtering by tag and exclude patterns!
return AddressesWithOrigins(
AddressWithOrigin(address=addr, origin=addr_to_origin[addr])
for addr in matched_addresses)
def test_address_specs_filter_by_exclude_pattern(
address_specs_rule_runner: RuleRunner) -> None:
address_specs_rule_runner.create_file("demo/f.txt")
address_specs_rule_runner.add_to_build_file(
"demo",
dedent("""\
mock_tgt(name="exclude_me", sources=["f.txt"])
mock_tgt(name="not_me", sources=["f.txt"])
"""),
)
bootstrapper = create_options_bootstrapper(
args=["--exclude-target-regexp=exclude_me.*"])
assert resolve_address_specs(address_specs_rule_runner,
[SiblingAddresses("demo")],
bootstrapper=bootstrapper) == {
AddressWithOrigin(
Address("demo", target_name="not_me"),
SiblingAddresses("demo"))
}
# The same filtering should work when given literal addresses, including file addresses.
# The filtering will operate against the normalized Address.spec.
literals_result = resolve_address_specs(
address_specs_rule_runner,
[
AddressLiteralSpec("demo", "exclude_me"),
AddressLiteralSpec("demo", "not_me"),
AddressLiteralSpec("demo/f.txt", "exclude_me"),
AddressLiteralSpec("demo/f.txt", "not_me"),
],
bootstrapper=bootstrapper,
)
assert literals_result == {
AddressWithOrigin(
Address("demo", relative_file_path="f.txt", target_name="not_me"),
AddressLiteralSpec("demo/f.txt", "not_me"),
),
AddressWithOrigin(Address("demo", target_name="not_me"),
AddressLiteralSpec("demo", "not_me")),
}
def test_filesystem_specs_literal_file(self) -> None:
self.create_files("demo", ["f1.txt", "f2.txt"])
self.add_to_build_file("demo", "target(sources=['*.txt'])")
spec = FilesystemLiteralSpec("demo/f1.txt")
result = self.request_single_product(
AddressesWithOrigins, Params(FilesystemSpecs([spec]), create_options_bootstrapper())
)
assert len(result) == 1
assert result[0] == AddressWithOrigin(
Address("demo", relative_file_path="f1.txt", target_name="demo"), origin=spec
)
async def addresses_with_origins_from_address_specs(
address_mapper: AddressMapper,
address_specs: AddressSpecs) -> AddressesWithOrigins:
"""Given an AddressMapper and list of AddressSpecs, return matching AddressesWithOrigins.
:raises: :class:`ResolveError` if there were no matching AddressFamilies or no targets
were matched.
"""
# Snapshot all BUILD files covered by the AddressSpecs, then group by directory.
snapshot = await Get(
Snapshot,
PathGlobs,
address_specs.to_path_globs(
build_patterns=address_mapper.build_patterns,
build_ignore_patterns=address_mapper.build_ignore_patterns,
),
)
dirnames = {os.path.dirname(f) for f in snapshot.files}
address_families = await MultiGet(
Get(AddressFamily, Dir(d)) for d in dirnames)
address_family_by_directory = {af.namespace: af for af in address_families}
matched_addresses: OrderedSet[Address] = OrderedSet()
addr_to_origin: Dict[Address, AddressSpec] = {}
for address_spec in address_specs:
# These may raise ResolveError, depending on the type of spec.
addr_families_for_spec = address_spec.matching_address_families(
address_family_by_directory)
addr_target_pairs_for_spec = address_spec.matching_addresses(
addr_families_for_spec)
if isinstance(address_spec,
SingleAddress) and not addr_target_pairs_for_spec:
addr_family = assert_single_element(addr_families_for_spec)
raise _did_you_mean_exception(addr_family, address_spec.name)
for addr, _ in addr_target_pairs_for_spec:
# A target might be covered by multiple specs, so we take the most specific one.
addr_to_origin[addr] = AddressSpecs.more_specific(
addr_to_origin.get(addr), address_spec)
matched_addresses.update(
addr for (addr, tgt) in addr_target_pairs_for_spec
if (address_specs.filter_by_global_options is False
or address_mapper.matches_filter_options(addr, tgt)))
return AddressesWithOrigins(
AddressWithOrigin(address=addr, origin=addr_to_origin[addr])
for addr in matched_addresses)
def test_resolve_addresses(self) -> None:
"""This tests that we correctly handle resolving from both address and filesystem specs."""
self.create_file("fs_spec/f.txt")
self.add_to_build_file("fs_spec", "target(sources=['f.txt'])")
self.create_file("address_spec/f.txt")
self.add_to_build_file("address_spec", "target(sources=['f.txt'])")
no_interaction_specs = ["fs_spec/f.txt", "address_spec:address_spec"]
# If a generated subtarget's original base target is included via an address spec,
# we will still include the generated subtarget for consistency. When we expand Targets
# into their base targets this redundancy is removed, but during Address expansion we
# get literal matches.
self.create_files("multiple_files", ["f1.txt", "f2.txt"])
self.add_to_build_file("multiple_files", "target(sources=['*.txt'])")
multiple_files_specs = ["multiple_files/f2.txt", "multiple_files:multiple_files"]
specs = SpecsCalculator.parse_specs([*no_interaction_specs, *multiple_files_specs])
result = self.request_single_product(
AddressesWithOrigins, Params(specs, create_options_bootstrapper())
)
assert set(result) == {
AddressWithOrigin(
Address("fs_spec", relative_file_path="f.txt"),
origin=FilesystemLiteralSpec("fs_spec/f.txt"),
),
AddressWithOrigin(
Address("address_spec"), origin=SingleAddress("address_spec", "address_spec"),
),
AddressWithOrigin(
Address("multiple_files"), origin=SingleAddress("multiple_files", "multiple_files"),
),
AddressWithOrigin(
Address("multiple_files", relative_file_path="f2.txt"),
origin=FilesystemLiteralSpec(file="multiple_files/f2.txt"),
),
}
def test_filesystem_specs_glob(self) -> None:
self.create_files("demo", ["f1.txt", "f2.txt"])
self.add_to_build_file("demo", "target(sources=['*.txt'])")
spec = FilesystemGlobSpec("demo/*.txt")
result = self.request_single_product(
AddressesWithOrigins, Params(FilesystemSpecs([spec]), create_options_bootstrapper()),
)
assert result == AddressesWithOrigins(
[
AddressWithOrigin(
Address("demo", relative_file_path="f1.txt", target_name="demo"), origin=spec
),
AddressWithOrigin(
Address("demo", relative_file_path="f2.txt", target_name="demo"), origin=spec
),
]
)
# If a glob and a literal spec both resolve to the same file, the literal spec should be
# used as it's more precise.
literal_spec = FilesystemLiteralSpec("demo/f1.txt")
result = self.request_single_product(
AddressesWithOrigins,
Params(FilesystemSpecs([spec, literal_spec]), create_options_bootstrapper()),
)
assert result == AddressesWithOrigins(
[
AddressWithOrigin(
Address("demo", relative_file_path="f1.txt", target_name="demo"),
origin=literal_spec,
),
AddressWithOrigin(
Address("demo", relative_file_path="f2.txt", target_name="demo"), origin=spec
),
]
)
async def addresses_with_origins_from_filesystem_specs(
filesystem_specs: FilesystemSpecs,
global_options: GlobalOptions,
) -> AddressesWithOrigins:
"""Find the owner(s) for each FilesystemSpec while preserving the original FilesystemSpec those
owners come from.
Every returned address will be a generated subtarget, meaning that each address will have
exactly one file in its `sources` field.
"""
owners_not_found_behavior = global_options.options.owners_not_found_behavior
paths_per_include = await MultiGet(
Get(
Paths,
PathGlobs,
filesystem_specs.path_globs_for_spec(
spec, owners_not_found_behavior.to_glob_match_error_behavior()
),
)
for spec in filesystem_specs.includes
)
owners_per_include = await MultiGet(
Get(Owners, OwnersRequest(sources=paths.files)) for paths in paths_per_include
)
addresses_to_specs: Dict[Address, FilesystemSpec] = {}
for spec, owners in zip(filesystem_specs.includes, owners_per_include):
if (
owners_not_found_behavior != OwnersNotFoundBehavior.ignore
and isinstance(spec, FilesystemLiteralSpec)
and not owners
):
_log_or_raise_unmatched_owners(
[PurePath(str(spec))],
global_options.options.owners_not_found_behavior,
ignore_option="--owners-not-found-behavior=ignore",
)
for address in owners:
# A target might be covered by multiple specs, so we take the most specific one.
addresses_to_specs[address] = FilesystemSpecs.more_specific(
addresses_to_specs.get(address), spec
)
return AddressesWithOrigins(
AddressWithOrigin(address, spec) for address, spec in addresses_to_specs.items()
)
async def addresses_with_origins_from_address_specs(
address_specs: AddressSpecs, global_options: GlobalOptions, specs_filter: AddressSpecsFilter
) -> AddressesWithOrigins:
"""Given an AddressMapper and list of AddressSpecs, return matching AddressesWithOrigins.
:raises: :class:`ResolveError` if the provided specs fail to match targets, and those spec
types expect to have matched something.
"""
matched_addresses: OrderedSet[Address] = OrderedSet()
addr_to_origin: Dict[Address, AddressSpec] = {}
filtering_disabled = address_specs.filter_by_global_options is False
# First convert all `AddressLiteralSpec`s. Some of the resulting addresses may be file
# addresses. This will raise an exception if any of the addresses are not valid.
literal_addresses = await MultiGet(
Get(Address, AddressInput(spec.path_component, spec.target_component))
for spec in address_specs.literals
)
literal_target_adaptors = await MultiGet(
Get(TargetAdaptor, Address, addr.maybe_convert_to_base_target())
for addr in literal_addresses
)
# We convert to targets for the side effect of validating that any file addresses actually
# belong to the specified base targets.
await Get(
UnexpandedTargets, Addresses(addr for addr in literal_addresses if not addr.is_base_target)
)
for literal_spec, addr, target_adaptor in zip(
address_specs.literals, literal_addresses, literal_target_adaptors
):
addr_to_origin[addr] = literal_spec
if filtering_disabled or specs_filter.matches(addr, target_adaptor):
matched_addresses.add(addr)
# Then, convert all `AddressGlobSpecs`. Snapshot all BUILD files covered by the specs, then
# group by directory.
snapshot = await Get(
Snapshot,
PathGlobs,
address_specs.to_path_globs(
build_patterns=global_options.options.build_patterns,
build_ignore_patterns=global_options.options.build_ignore,
),
)
dirnames = {os.path.dirname(f) for f in snapshot.files}
address_families = await MultiGet(Get(AddressFamily, Dir(d)) for d in dirnames)
address_family_by_directory = {af.namespace: af for af in address_families}
for glob_spec in address_specs.globs:
# These may raise ResolveError, depending on the type of spec.
addr_families_for_spec = glob_spec.matching_address_families(address_family_by_directory)
addr_target_pairs_for_spec = glob_spec.matching_addresses(addr_families_for_spec)
for addr, _ in addr_target_pairs_for_spec:
# A target might be covered by multiple specs, so we take the most specific one.
addr_to_origin[addr] = AddressSpecs.more_specific(addr_to_origin.get(addr), glob_spec)
matched_addresses.update(
addr
for (addr, tgt) in addr_target_pairs_for_spec
if filtering_disabled or specs_filter.matches(addr, tgt)
)
return AddressesWithOrigins(
AddressWithOrigin(address=addr, origin=addr_to_origin[addr]) for addr in matched_addresses
)