def test_fails_on_nonexistent_specs(self) -> None:
"""Test that address specs referring to nonexistent targets raise a ResolveError."""
address_family = AddressFamily('root',
{'a': ('root/BUILD', TargetAdaptor())})
address_specs = AddressSpecs(
[SingleAddress('root', 'b'),
SingleAddress('root', 'a')])
expected_rx_str = re.escape(
""""b" was not found in namespace "root". Did you mean one of:
:a""")
with self.assertRaisesRegex(ResolveError, expected_rx_str):
self._resolve_build_file_addresses(address_specs, address_family,
self._snapshot(),
self._address_mapper())
# Ensure that we still catch nonexistent targets later on in the list of command-line
# address specs.
address_specs = AddressSpecs(
[SingleAddress('root', 'a'),
SingleAddress('root', 'b')])
with self.assertRaisesRegex(ResolveError, expected_rx_str):
self._resolve_build_file_addresses(address_specs, address_family,
self._snapshot(),
self._address_mapper())
async def find_owners(owners_request: OwnersRequest) -> Owners:
# Determine which of the sources are live and which are deleted.
sources_set_snapshot = await Get(Snapshot,
PathGlobs(owners_request.sources))
live_files = FrozenOrderedSet(sources_set_snapshot.files)
deleted_files = FrozenOrderedSet(s for s in owners_request.sources
if s not in live_files)
live_dirs = FrozenOrderedSet(os.path.dirname(s) for s in live_files)
deleted_dirs = FrozenOrderedSet(os.path.dirname(s) for s in deleted_files)
matching_addresses: OrderedSet[Address] = OrderedSet()
unmatched_sources = set(owners_request.sources)
for live in (True, False):
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
# For live files, we use expanded Targets, which have file level precision but which are
# only created for existing files. For deleted files we use UnexpandedTargets, which have
# the original declared glob.
candidate_targets: Iterable[Target]
if live:
if not live_dirs:
continue
sources_set = live_files
candidate_specs = tuple(
AscendantAddresses(directory=d) for d in live_dirs)
candidate_targets = await Get(Targets,
AddressSpecs(candidate_specs))
else:
if not deleted_dirs:
continue
sources_set = deleted_files
candidate_specs = tuple(
AscendantAddresses(directory=d) for d in deleted_dirs)
candidate_targets = await Get(UnexpandedTargets,
AddressSpecs(candidate_specs))
build_file_addresses = await MultiGet(
Get(BuildFileAddress, Address, tgt.address)
for tgt in candidate_targets)
for candidate_tgt, bfa in zip(candidate_targets, build_file_addresses):
matching_files = set(
matches_filespec(candidate_tgt.get(Sources).filespec,
paths=sources_set))
if not matching_files and bfa.rel_path not in sources_set:
continue
unmatched_sources -= matching_files
matching_addresses.add(candidate_tgt.address)
if (unmatched_sources and owners_request.owners_not_found_behavior !=
OwnersNotFoundBehavior.ignore):
_log_or_raise_unmatched_owners(
[PurePath(path) for path in unmatched_sources],
owners_request.owners_not_found_behavior)
return Owners(matching_addresses)
async def find_owners(build_configuration: BuildConfiguration,
address_mapper: AddressMapper,
owners_request: OwnersRequest) -> BuildFileAddresses:
sources_set = OrderedSet(owners_request.sources)
dirs_set = OrderedSet(dirname(source) for source in sources_set)
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set)
candidate_targets = await Get[HydratedTargets](
AddressSpecs(candidate_specs))
# Match the source globs against the expanded candidate targets.
def owns_any_source(legacy_target):
"""Given a `HydratedTarget` instance, check if it owns the given source file."""
target_kwargs = legacy_target.adaptor.kwargs()
# Handle `sources`-declaring targets.
# NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target,
# so we don't actually call `fileset.matches` here.
# TODO: This matching logic should be implemented using the rust `fs` crate for two reasons:
# 1) having two implementations isn't great
# 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match
target_sources = target_kwargs.get('sources', None)
if target_sources and any_matches_filespec(
paths=sources_set, spec=target_sources.filespec):
return True
return False
direct_owners = tuple(
ht.adaptor.address for ht in candidate_targets
if LegacyAddressMapper.any_is_declaring_file(
ht.adaptor.address, sources_set) or owns_any_source(ht))
# If the OwnersRequest does not require dependees, then we're done.
if owners_request.include_dependees == IncludeDependeesOption.NONE:
return BuildFileAddresses(direct_owners)
# Otherwise: find dependees.
all_addresses = await Get[BuildFileAddresses](AddressSpecs(
(DescendantAddresses(''), )))
all_structs = [
s.value
for s in await MultiGet(Get[HydratedStruct](Address, a.to_address())
for a in all_addresses)
]
bfa = build_configuration.registered_aliases()
graph = _DependentGraph.from_iterable(
target_types_from_build_file_aliases(bfa), address_mapper, all_structs)
if owners_request.include_dependees == IncludeDependeesOption.DIRECT:
return BuildFileAddresses(
tuple(graph.dependents_of_addresses(direct_owners)))
assert owners_request.include_dependees == IncludeDependeesOption.TRANSITIVE
return BuildFileAddresses(
tuple(graph.transitive_dependents_of_addresses(direct_owners)))
async def addresses_with_origins_from_address_specs(
address_mapper: AddressMapper,
address_specs: AddressSpecs) -> AddressesWithOrigins:
"""Given an AddressMapper and list of AddressSpecs, return matching AddressesWithOrigins.
:raises: :class:`ResolveError` if there were no matching AddressFamilies or no targets
were matched.
"""
# Snapshot all BUILD files covered by the AddressSpecs, then group by directory.
snapshot = await Get(
Snapshot,
PathGlobs,
address_specs.to_path_globs(
build_patterns=address_mapper.build_patterns,
build_ignore_patterns=address_mapper.build_ignore_patterns,
),
)
dirnames = {os.path.dirname(f) for f in snapshot.files}
address_families = await MultiGet(
Get(AddressFamily, Dir(d)) for d in dirnames)
address_family_by_directory = {af.namespace: af for af in address_families}
matched_addresses: OrderedSet[Address] = OrderedSet()
addr_to_origin: Dict[Address, AddressSpec] = {}
for address_spec in address_specs:
# These may raise ResolveError, depending on the type of spec.
addr_families_for_spec = address_spec.matching_address_families(
address_family_by_directory)
addr_target_pairs_for_spec = address_spec.matching_addresses(
addr_families_for_spec)
if isinstance(address_spec,
SingleAddress) and not addr_target_pairs_for_spec:
addr_family = assert_single_element(addr_families_for_spec)
raise _did_you_mean_exception(addr_family, address_spec.name)
for addr, _ in addr_target_pairs_for_spec:
# A target might be covered by multiple specs, so we take the most specific one.
addr_to_origin[addr] = AddressSpecs.more_specific(
addr_to_origin.get(addr), address_spec)
matched_addresses.update(
addr for (addr, tgt) in addr_target_pairs_for_spec
if (address_specs.filter_by_global_options is False
or address_mapper.matches_filter_options(addr, tgt)))
return AddressesWithOrigins(
AddressWithOrigin(address=addr, origin=addr_to_origin[addr])
for addr in matched_addresses)
async def map_protobuf_to_python_modules(
_: PythonProtobufMappingMarker,
) -> FirstPartyPythonMappingImpl:
all_expanded_targets = await Get(Targets, AddressSpecs([DescendantAddresses("")]))
protobuf_targets = tuple(tgt for tgt in all_expanded_targets if tgt.has_field(ProtobufSources))
stripped_sources_per_target = await MultiGet(
Get(StrippedSourceFileNames, SourcesPathsRequest(tgt[ProtobufSources]))
for tgt in protobuf_targets
)
modules_to_addresses: Dict[str, Tuple[Address]] = {}
modules_with_multiple_owners: Set[str] = set()
def add_module(module: str, tgt: Target) -> None:
if module in modules_to_addresses:
modules_with_multiple_owners.add(module)
else:
modules_to_addresses[module] = (tgt.address,)
for tgt, stripped_sources in zip(protobuf_targets, stripped_sources_per_target):
for stripped_f in stripped_sources:
# NB: We don't consider the MyPy plugin, which generates `_pb2.pyi`. The stubs end up
# sharing the same module as the implementation `_pb2.py`. Because both generated files
# come from the same original Protobuf target, we're covered.
add_module(proto_path_to_py_module(stripped_f, suffix="_pb2"), tgt)
if tgt.get(ProtobufGrpcToggle).value:
add_module(proto_path_to_py_module(stripped_f, suffix="_pb2_grpc"), tgt)
# Remove modules with ambiguous owners.
for ambiguous_module in modules_with_multiple_owners:
modules_to_addresses.pop(ambiguous_module)
return FirstPartyPythonMappingImpl(modules_to_addresses)
def inject_addresses_closure(self, addresses):
addresses = set(addresses) - set(self._target_by_address.keys())
if not addresses:
return
dependencies = (SingleAddress(directory=a.spec_path, name=a.target_name) for a in addresses)
for _ in self._inject_address_specs(AddressSpecs(dependencies)):
pass
async def rename_conflicting_targets(
ptgts: PutativeTargets) -> UniquelyNamedPutativeTargets:
"""Ensure that no target addresses collide."""
all_existing_tgts = await Get(
UnexpandedTargets, AddressSpecs([MaybeEmptyDescendantAddresses("")]))
existing_addrs: Set[str] = {tgt.address.spec for tgt in all_existing_tgts}
uniquely_named_putative_targets: List[PutativeTarget] = []
for ptgt in ptgts:
if not ptgt.addressable:
# Non-addressable PutativeTargets never have collision issues.
uniquely_named_putative_targets.append(ptgt)
continue
idx = 0
possibly_renamed_ptgt = ptgt
# Targets in root-level BUILD files must be named explicitly.
if possibly_renamed_ptgt.path == "" and possibly_renamed_ptgt.kwargs.get(
"name") is None:
possibly_renamed_ptgt = possibly_renamed_ptgt.rename("root")
# Eliminate any address collisions.
while possibly_renamed_ptgt.address.spec in existing_addrs:
possibly_renamed_ptgt = ptgt.rename(f"{ptgt.name}{idx}")
idx += 1
uniquely_named_putative_targets.append(possibly_renamed_ptgt)
existing_addrs.add(possibly_renamed_ptgt.address.spec)
return UniquelyNamedPutativeTargets(
PutativeTargets(uniquely_named_putative_targets))
def parse_specs(
cls,
raw_specs: Iterable[str],
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> Specs:
"""Parse raw string specs into a Specs object."""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
address_specs: OrderedSet[AddressSpec] = OrderedSet()
filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet()
for spec_str in raw_specs:
parsed_spec = spec_parser.parse_spec(spec_str)
if isinstance(parsed_spec, AddressSpec):
address_specs.add(parsed_spec)
else:
filesystem_specs.add(parsed_spec)
address_specs_collection = AddressSpecs(
dependencies=address_specs,
exclude_patterns=exclude_patterns if exclude_patterns else tuple(),
tags=tags,
)
filesystem_specs_collection = FilesystemSpecs(filesystem_specs)
return Specs(
address_specs=address_specs_collection,
filesystem_specs=filesystem_specs_collection,
)
async def setup_pytest_lockfile(
_: PytestLockfileSentinel, pytest: PyTest,
python_setup: PythonSetup) -> PythonLockfileRequest:
if not pytest.uses_lockfile:
return PythonLockfileRequest.from_tool(pytest)
# Even though we run each python_tests target in isolation, we need a single lockfile that
# works with them all (and their transitive deps).
#
# This first computes the constraints for each individual `python_tests` target
# (which will AND across each target in the closure). Then, it ORs all unique resulting
# interpreter constraints. The net effect is that every possible Python interpreter used will
# be covered.
all_build_targets = await Get(UnexpandedTargets,
AddressSpecs([DescendantAddresses("")]))
transitive_targets_per_test = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_build_targets if PythonTestFieldSet.is_applicable(tgt))
unique_constraints = {
InterpreterConstraints.create_from_targets(transitive_targets.closure,
python_setup)
for transitive_targets in transitive_targets_per_test
}
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return PythonLockfileRequest.from_tool(
pytest, constraints
or InterpreterConstraints(python_setup.interpreter_constraints))
async def map_first_party_modules_to_addresses(
) -> FirstPartyModuleToAddressMapping:
all_expanded_targets = await Get(Targets,
AddressSpecs([DescendantAddresses("")]))
candidate_targets = tuple(tgt for tgt in all_expanded_targets
if tgt.has_field(PythonSources))
stripped_sources_per_explicit_target = await MultiGet(
Get(StrippedSourceFiles, SourceFilesRequest([tgt[PythonSources]]))
for tgt in candidate_targets)
modules_to_addresses: Dict[str, Address] = {}
modules_with_multiple_owners: Set[str] = set()
for tgt, stripped_sources in zip(candidate_targets,
stripped_sources_per_explicit_target):
for stripped_f in stripped_sources.snapshot.files:
module = PythonModule.create_from_stripped_path(
PurePath(stripped_f)).module
if module in modules_to_addresses:
modules_with_multiple_owners.add(module)
else:
modules_to_addresses[module] = tgt.address
# Remove modules with ambiguous owners.
for module in modules_with_multiple_owners:
modules_to_addresses.pop(module)
return FirstPartyModuleToAddressMapping(
FrozenDict(sorted(modules_to_addresses.items())))
async def inject_docker_dependencies(request: InjectDockerDependencies) -> InjectedDependencies:
"""Inspects COPY instructions in the Dockerfile for references to known packagable targets."""
dockerfile_info = await Get(
DockerfileInfo, DockerfileInfoRequest(request.dependencies_field.address)
)
# Parse all putative target addresses.
putative_addresses = await Get(
Addresses,
UnparsedAddressInputs(
dockerfile_info.putative_target_addresses,
owning_address=dockerfile_info.address,
),
)
# Get the target for those addresses that are known.
directories = {address.spec_path for address in putative_addresses}
all_addresses = await Get(Addresses, AddressSpecs(map(MaybeEmptySiblingAddresses, directories)))
targets = await Get(
Targets, Addresses((address for address in putative_addresses if address in all_addresses))
)
# Only keep those targets that we can "package".
package = await Get(FieldSetsPerTarget, FieldSetsPerTargetRequest(PackageFieldSet, targets))
referenced_targets = (
field_sets[0].address for field_sets in package.collection if len(field_sets) > 0
)
return InjectedDependencies(Addresses(referenced_targets))
async def setup_mypy_lockfile(
_: MyPyLockfileSentinel,
first_party_plugins: MyPyFirstPartyPlugins,
mypy: MyPy,
python_setup: PythonSetup,
) -> PythonLockfileRequest:
if not mypy.uses_lockfile:
return PythonLockfileRequest.from_tool(mypy)
constraints = mypy.interpreter_constraints
if mypy.options.is_default("interpreter_constraints"):
all_build_targets = await Get(UnexpandedTargets,
AddressSpecs([DescendantAddresses("")]))
all_transitive_targets = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_build_targets if MyPyFieldSet.is_applicable(tgt))
unique_constraints = {
InterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup)
for transitive_targets in all_transitive_targets
}
code_constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
if code_constraints.requires_python38_or_newer(
python_setup.interpreter_universe):
constraints = code_constraints
return PythonLockfileRequest.from_tool(
mypy,
constraints,
extra_requirements=first_party_plugins.requirement_strings)
async def map_third_party_modules_to_addresses(
) -> ThirdPartyPythonModuleMapping:
all_targets = await Get(Targets, AddressSpecs([DescendantAddresses("")]))
modules_to_addresses: dict[str, Address] = {}
modules_with_multiple_owners: DefaultDict[str,
set[Address]] = defaultdict(set)
for tgt in all_targets:
if not tgt.has_field(PythonRequirementsField):
continue
module_map = tgt.get(ModuleMappingField).value
for python_req in tgt[PythonRequirementsField].value:
modules = module_map.get(
python_req.project_name,
[python_req.project_name.lower().replace("-", "_")],
)
for module in modules:
if module in modules_to_addresses:
modules_with_multiple_owners[module].update(
{modules_to_addresses[module], tgt.address})
else:
modules_to_addresses[module] = tgt.address
# Remove modules with ambiguous owners.
for module in modules_with_multiple_owners:
modules_to_addresses.pop(module)
return ThirdPartyPythonModuleMapping(
mapping=FrozenDict(sorted(modules_to_addresses.items())),
ambiguous_modules=FrozenDict(
(k, tuple(sorted(v)))
for k, v in sorted(modules_with_multiple_owners.items())),
)
async def setup_ipython_lockfile(
_: IPythonLockfileSentinel, ipython: IPython, python_setup: PythonSetup
) -> PythonLockfileRequest:
if not ipython.uses_lockfile:
return PythonLockfileRequest.from_tool(ipython)
# IPython is often run against the whole repo (`./pants repl ::`), but it is possible to run
# on subsets of the codebase with disjoint interpreter constraints, such as
# `./pants repl py2::` and then `./pants repl py3::`. Still, even with those subsets possible,
# we need a single lockfile that works with all possible Python interpreters in use.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_build_targets = await Get(UnexpandedTargets, AddressSpecs([DescendantAddresses("")]))
unique_constraints = {
InterpreterConstraints.create_from_compatibility_fields(
[tgt[InterpreterConstraintsField]], python_setup
)
for tgt in all_build_targets
if tgt.has_field(InterpreterConstraintsField)
}
constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints))
return PythonLockfileRequest.from_tool(
ipython, constraints or InterpreterConstraints(python_setup.interpreter_constraints)
)
async def map_third_party_modules_to_addresses(
) -> ThirdPartyModuleToAddressMapping:
all_targets = await Get(Targets, AddressSpecs([DescendantAddresses("")]))
modules_to_addresses: Dict[str, Address] = {}
modules_with_multiple_owners: Set[str] = set()
for tgt in all_targets:
if not tgt.has_field(PythonRequirementsField):
continue
module_map = tgt.get(ModuleMappingField).value or {
} # type: ignore[var-annotated]
for python_req in tgt[PythonRequirementsField].value:
modules = module_map.get(
python_req.project_name,
[python_req.project_name.lower().replace("-", "_")],
)
for module in modules:
if module in modules_to_addresses:
modules_with_multiple_owners.add(module)
else:
modules_to_addresses[module] = tgt.address
# Remove modules with ambiguous owners.
for module in modules_with_multiple_owners:
modules_to_addresses.pop(module)
return ThirdPartyModuleToAddressMapping(
FrozenDict(sorted(modules_to_addresses.items())))
def test_get_target_data(rule_runner: RuleRunner) -> None:
rule_runner.write_files({
"foo/BUILD":
dedent("""\
target(name="bar", dependencies=[":baz"])
files(name="baz", sources=["*.txt"])
"""),
"foo/a.txt":
"",
"foo/b.txt":
"",
})
tds = rule_runner.request(TargetDatas,
[AddressSpecs([DescendantAddresses("foo")])])
assert tds == TargetDatas([
TargetData(
GenericTarget({"dependencies": [":baz"]},
Address("foo", target_name="bar")),
None,
("foo:baz", ),
),
TargetData(
FilesGeneratorTarget({"sources": ["*.txt"]},
Address("foo", target_name="baz")),
("foo/a.txt", "foo/b.txt"),
tuple(),
),
])
async def map_shell_files() -> ShellMapping:
all_expanded_targets = await Get(Targets,
AddressSpecs([DescendantAddresses("")]))
shell_tgts = tuple(tgt for tgt in all_expanded_targets
if tgt.has_field(ShellSources))
sources_per_target = await MultiGet(
Get(SourcesPaths, SourcesPathsRequest(tgt[ShellSources]))
for tgt in shell_tgts)
files_to_addresses: dict[str, Address] = {}
files_with_multiple_owners: DefaultDict[str,
set[Address]] = defaultdict(set)
for tgt, sources in zip(shell_tgts, sources_per_target):
for f in sources.files:
if f in files_to_addresses:
files_with_multiple_owners[f].update(
{files_to_addresses[f], tgt.address})
else:
files_to_addresses[f] = tgt.address
# Remove files with ambiguous owners.
for ambiguous_f in files_with_multiple_owners:
files_to_addresses.pop(ambiguous_f)
return ShellMapping(
mapping=FrozenDict(sorted(files_to_addresses.items())),
ambiguous_modules=FrozenDict(
(k, tuple(sorted(v)))
for k, v in sorted(files_with_multiple_owners.items())),
)
async def find_owners(owners_request: OwnersRequest) -> Owners:
sources_set = FrozenOrderedSet(owners_request.sources)
dirs_set = FrozenOrderedSet(
os.path.dirname(source) for source in sources_set)
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set)
candidate_targets = await Get[HydratedTargets](
AddressSpecs(candidate_specs))
# Match the source globs against the expanded candidate targets.
def owns_any_source(legacy_target: HydratedTarget) -> bool:
"""Given a `HydratedTarget` instance, check if it owns the given source file."""
target_kwargs = legacy_target.adaptor.kwargs()
# Handle `sources`-declaring targets.
# NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target,
# so we don't actually call `fileset.matches` here.
# TODO: This matching logic should be implemented using the rust `fs` crate for two reasons:
# 1) having two implementations isn't great
# 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match
target_sources = target_kwargs.get("sources", None)
return target_sources and any_matches_filespec(
paths=sources_set, spec=target_sources.filespec)
build_file_addresses = await MultiGet(
Get[BuildFileAddress](Address, ht.adaptor.address)
for ht in candidate_targets)
owners = Addresses(
ht.adaptor.address
for ht, bfa in zip(candidate_targets, build_file_addresses)
if LegacyAddressMapper.any_is_declaring_file(bfa, sources_set)
or owns_any_source(ht))
return Owners(owners)
async def find_nearest_go_module(
request: FindNearestGoModuleRequest) -> ResolvedOwningGoModule:
spec_path = request.spec_path
candidate_targets = await Get(
UnexpandedTargets,
AddressSpecs([
AscendantAddresses(spec_path),
MaybeEmptySiblingAddresses(spec_path)
]),
)
go_module_targets = [
tgt for tgt in candidate_targets if tgt.has_field(GoModuleSources)
]
# Sort by address.spec_path in descending order so the nearest go_module target is sorted first.
sorted_go_module_targets = sorted(go_module_targets,
key=lambda tgt: tgt.address.spec_path,
reverse=True)
if sorted_go_module_targets:
nearest_go_module_target = sorted_go_module_targets[0]
return ResolvedOwningGoModule(
module_address=nearest_go_module_target.address)
else:
# TODO: Consider eventually requiring all go_package's to associate with a go_module.
return ResolvedOwningGoModule(module_address=None)
async def infer_terraform_module_dependencies(
request: InferTerraformModuleDependenciesRequest,
) -> InferredDependencies:
hydrated_sources = await Get(HydratedSources,
HydrateSourcesRequest(request.sources_field))
paths = OrderedSet(filename for filename in hydrated_sources.snapshot.files
if filename.endswith(".tf"))
result = await Get(
ProcessResult,
ParseTerraformModuleSources(
sources_digest=hydrated_sources.snapshot.digest,
paths=tuple(paths),
),
)
candidate_spec_paths = [
line for line in result.stdout.decode("utf-8").split("\n") if line
]
# For each path, see if there is a `terraform_module` target at the specified spec_path.
candidate_targets = await Get(
Targets,
AddressSpecs([
MaybeEmptySiblingAddresses(path) for path in candidate_spec_paths
]))
# TODO: Need to either implement the standard ambiguous dependency logic or ban >1 terraform_module
# per directory.
terraform_module_addresses = [
tgt.address for tgt in candidate_targets
if tgt.has_field(TerraformModuleSourcesField)
]
return InferredDependencies(terraform_module_addresses)
def resolve_address_specs(
rule_runner: RuleRunner,
specs: Iterable[AddressSpec],
) -> Set[Address]:
result = rule_runner.request(
Addresses, [AddressSpecs(specs, filter_by_global_options=True)])
return set(result)
async def find_owners(
build_configuration: BuildConfiguration,
address_mapper: AddressMapper,
changed_request: ChangedRequest,
) -> ChangedAddresses:
owners = await Get[Owners](OwnersRequest(sources=changed_request.sources))
# If the ChangedRequest does not require dependees, then we're done.
if changed_request.include_dependees == IncludeDependeesOption.NONE:
return ChangedAddresses(owners.addresses)
# Otherwise: find dependees.
all_addresses = await Get[Addresses](AddressSpecs(
(DescendantAddresses(""), )))
all_structs = [
s.value for s in await MultiGet(Get[HydratedStruct](Address, a)
for a in all_addresses)
]
bfa = build_configuration.registered_aliases()
graph = _DependentGraph.from_iterable(
target_types_from_build_file_aliases(bfa), address_mapper, all_structs)
if changed_request.include_dependees == IncludeDependeesOption.DIRECT:
return ChangedAddresses(
Addresses(graph.dependents_of_addresses(owners.addresses)))
return ChangedAddresses(
Addresses(graph.transitive_dependents_of_addresses(owners.addresses)))
async def map_protobuf_files() -> ProtobufMapping:
all_expanded_targets = await Get(Targets, AddressSpecs([DescendantAddresses("")]))
protobuf_targets = tuple(tgt for tgt in all_expanded_targets if tgt.has_field(ProtobufSources))
stripped_sources_per_target = await MultiGet(
Get(StrippedSourceFileNames, SourcesPathsRequest(tgt[ProtobufSources]))
for tgt in protobuf_targets
)
stripped_files_to_addresses: dict[str, Address] = {}
stripped_files_with_multiple_owners: DefaultDict[str, set[Address]] = defaultdict(set)
for tgt, stripped_sources in zip(protobuf_targets, stripped_sources_per_target):
for stripped_f in stripped_sources:
if stripped_f in stripped_files_to_addresses:
stripped_files_with_multiple_owners[stripped_f].update(
{stripped_files_to_addresses[stripped_f], tgt.address}
)
else:
stripped_files_to_addresses[stripped_f] = tgt.address
# Remove files with ambiguous owners.
for ambiguous_stripped_f in stripped_files_with_multiple_owners:
stripped_files_to_addresses.pop(ambiguous_stripped_f)
return ProtobufMapping(
mapping=FrozenDict(sorted(stripped_files_to_addresses.items())),
ambiguous_modules=FrozenDict(
(k, tuple(sorted(v))) for k, v in sorted(stripped_files_with_multiple_owners.items())
),
)
def _internal_scan_specs(self,
specs,
fail_fast=True,
missing_is_fatal=True):
# TODO: This should really use `product_request`, but on the other hand, we need to
# deprecate the entire `AddressMapper` interface anyway. See #4769.
request = self._scheduler.execution_request(
[BuildFileAddresses], [AddressSpecs(tuple(specs))])
returns, throws = self._scheduler.execute(request)
if throws:
_, state = throws[0]
if isinstance(state.exc, (AddressLookupError, ResolveError)):
if missing_is_fatal:
raise self.BuildFileScanError(
"AddressSpec `{}` does not match any targets.\n{}".
format(self._specs_string(specs), str(state.exc)))
else:
# NB: ignore Throws containing ResolveErrors because they are due to missing targets / files
return set()
else:
raise self.BuildFileScanError(str(state.exc))
_, state = returns[0]
if missing_is_fatal and not state.value.dependencies:
raise self.BuildFileScanError(
"AddressSpec `{}` does not match any targets.".format(
self._specs_string(specs)))
return set(state.value.dependencies)
def _init_engine(self, local_store_dir: Optional[str] = None) -> None:
if self._scheduler is not None:
return
options_bootstrapper = OptionsBootstrapper.create(
args=["--pants-config-files=[]"])
local_store_dir = (local_store_dir
or options_bootstrapper.bootstrap_options.
for_global_scope().local_store_dir)
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=[],
local_store_dir=local_store_dir,
build_file_imports_behavior=BuildFileImportsBehavior.error,
native=init_native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config(),
build_ignore_patterns=None,
).new_session(zipkin_trace_v2=False, build_id="buildid_for_test")
self._scheduler = graph_session.scheduler_session
self._build_graph, self._address_mapper = graph_session.create_build_graph(
Specs(address_specs=AddressSpecs([]),
filesystem_specs=FilesystemSpecs([])),
self._build_root(),
)
async def restrict_conflicting_sources(
ptgt: PutativeTarget) -> DisjointSourcePutativeTarget:
source_paths = await Get(
Paths,
PathGlobs(
Sources.prefix_glob_with_dirpath(ptgt.path, glob)
for glob in ptgt.owned_sources),
)
source_path_set = set(source_paths.files)
source_dirs = {os.path.dirname(path) for path in source_path_set}
possible_owners = await Get(
UnexpandedTargets,
AddressSpecs(AscendantAddresses(d) for d in source_dirs))
possible_owners_sources = await MultiGet(
Get(SourcesPaths, SourcesPathsRequest(t.get(Sources)))
for t in possible_owners)
conflicting_targets = []
for tgt, sources in zip(possible_owners, possible_owners_sources):
if source_path_set.intersection(sources.files):
conflicting_targets.append(tgt)
if conflicting_targets:
conflicting_addrs = sorted(tgt.address.spec
for tgt in conflicting_targets)
explicit_srcs_str = ", ".join(ptgt.kwargs.get("sources")
or []) # type: ignore[arg-type]
orig_sources_str = (f"[{explicit_srcs_str}]" if explicit_srcs_str else
f"the default for {ptgt.type_alias}")
ptgt = ptgt.restrict_sources().add_comments([
f"# NOTE: Sources restricted from {orig_sources_str} due to conflict with"
] + [f"# - {caddr}" for caddr in conflicting_addrs])
return DisjointSourcePutativeTarget(ptgt)
def inject_address_specs_closure(
self,
address_specs: Iterable[AddressSpec],
fail_fast=None,
) -> Iterator[BuildFileAddress]:
# Request loading of these address specs.
for address in self._inject_address_specs(AddressSpecs(address_specs)):
yield address
def create(
cls,
options_bootstrapper: OptionsBootstrapper,
options: Options,
session: SchedulerSession,
build_root: Optional[str] = None,
) -> Specs:
specs = cls.parse_specs(raw_specs=options.specs, build_root=build_root)
changed_options = ChangedOptions.from_options(
options.for_scope("changed"))
logger.debug("specs are: %s", specs)
logger.debug("changed_options are: %s", changed_options)
if specs.provided and changed_options.provided:
changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
if specs.filesystem_specs and specs.address_specs:
specs_description = "target and file arguments"
elif specs.filesystem_specs:
specs_description = "file arguments"
else:
specs_description = "target arguments"
raise InvalidSpecConstraint(
f"You used `{changed_name}` at the same time as using {specs_description}. Please "
"use only one.")
if not changed_options.provided:
return specs
scm = get_scm()
if not scm:
raise InvalidSpecConstraint(
"The `--changed-*` options are not available without a recognized SCM (usually "
"Git).")
changed_request = ChangedRequest(
sources=tuple(changed_options.changed_files(scm=scm)),
dependees=changed_options.dependees,
)
(changed_addresses, ) = session.product_request(
ChangedAddresses, [Params(changed_request, options_bootstrapper)])
logger.debug("changed addresses: %s", changed_addresses)
address_specs = []
filesystem_specs = []
for address in cast(ChangedAddresses, changed_addresses):
if not address.is_base_target:
# TODO: Should adjust Specs parsing to support parsing the disambiguated file
# Address, which would bypass-rediscovering owners.
filesystem_specs.append(FilesystemLiteralSpec(
address.filename))
else:
address_specs.append(
SingleAddress(address.spec_path, address.target_name))
return Specs(
AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs(filesystem_specs),
)
def __init__(
self,
root_dir: str,
options_bootstrapper: OptionsBootstrapper,
options: Options,
build_config: BuildConfiguration,
run_tracker: RunTracker,
reporting: Reporting,
graph_session: LegacyGraphSession,
specs: Specs,
exiter=sys.exit,
) -> None:
"""
:param root_dir: The root directory of the pants workspace (aka the "build root").
:param options: The global, pre-initialized Options instance.
:param build_config: A pre-initialized BuildConfiguration instance.
:param run_tracker: The global, pre-initialized/running RunTracker instance.
:param reporting: The global, pre-initialized Reporting instance.
:param graph_session: The graph session for this run.
:param specs: The specs for this run, i.e. either the address or filesystem specs.
:param func exiter: A function that accepts an exit code value and exits. (for tests, Optional)
"""
self._root_dir = root_dir
self._options_bootstrapper = options_bootstrapper
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._graph_session = graph_session
self._specs = specs
self._exiter = exiter
self._global_options = options.for_global_scope()
self._fail_fast = self._global_options.fail_fast
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
# V1 tasks do not understand FilesystemSpecs, so we eagerly convert them into AddressSpecs.
if self._specs.filesystem_specs.dependencies:
(owned_addresses,
) = self._graph_session.scheduler_session.product_request(
Addresses, [
Params(self._specs.filesystem_specs,
self._options_bootstrapper)
])
updated_address_specs = AddressSpecs(
dependencies=tuple(
SingleAddress(a.spec_path, a.target_name)
for a in owned_addresses),
tags=self._specs.address_specs.matcher.tags,
exclude_patterns=self._specs.address_specs.matcher.
exclude_patterns,
)
self._specs = Specs(
address_specs=updated_address_specs,
filesystem_specs=FilesystemSpecs([]),
)
async def determine_all_owned_sources() -> AllOwnedSources:
all_tgts = await Get(UnexpandedTargets,
AddressSpecs([MaybeEmptyDescendantAddresses("")]))
all_sources_paths = await MultiGet(
Get(SourcesPaths, SourcesPathsRequest(tgt.get(Sources)))
for tgt in all_tgts)
return AllOwnedSources(
itertools.chain.from_iterable(sources_paths.files
for sources_paths in all_sources_paths))
