async def parse_python_imports(request: ParsePythonImportsRequest) -> ParsedPythonImports:
python_interpreter, script_digest, stripped_sources = await MultiGet(
Get(PythonExecutable, PexInterpreterConstraints, request.interpreter_constraints),
Get(Digest, CreateDigest([FileContent("__parse_python_imports.py", _SCRIPT.encode())])),
Get(StrippedSourceFiles, SourceFilesRequest([request.sources])),
)
input_digest = await Get(
Digest, MergeDigests([script_digest, stripped_sources.snapshot.digest])
)
process_result = await Get(
ProcessResult,
Process(
argv=[
python_interpreter.path,
"./__parse_python_imports.py",
*stripped_sources.snapshot.files,
],
input_digest=input_digest,
description=f"Determine Python imports for {request.sources.address}",
level=LogLevel.DEBUG,
),
)
explicit_imports, _, string_imports = process_result.stdout.decode().partition("--")
return ParsedPythonImports(
explicit_imports=FrozenOrderedSet(explicit_imports.strip().splitlines()),
string_imports=FrozenOrderedSet(string_imports.strip().splitlines()),
)
def find_artifact_mapping(
import_name: str,
mapping: ThirdPartyPackageToArtifactMapping,
available_artifacts: AvailableThirdPartyArtifacts,
) -> FrozenOrderedSet[Address]:
imp_parts = import_name.split(".")
current_node = mapping.mapping_root
found_nodes = []
for imp_part in imp_parts:
child_node_opt = current_node.find_child(imp_part)
if not child_node_opt:
break
found_nodes.append(child_node_opt)
current_node = child_node_opt
if not found_nodes:
return FrozenOrderedSet()
# If the length of the found nodes equals the number of parts of the package path, then there
# is an exact match.
if len(found_nodes) == len(imp_parts):
addresses = available_artifacts.addresses_for_coordinates(
found_nodes[-1].coordinates)
return addresses
# Otherwise, check for the first found node (in reverse order) to match recursively, and use its coordinate.
for found_node in reversed(found_nodes):
if found_node.recursive:
addresses = available_artifacts.addresses_for_coordinates(
found_node.coordinates)
return addresses
# Nothing matched so return no match.
return FrozenOrderedSet()
def test_target_types(self):
def target_types():
return [DummyTarget, DummyTarget2]
with self.create_register(
target_types=target_types) as backend_package:
load_backend(self.bc_builder, backend_package, is_v1_backend=True)
assert self.bc_builder.create().target_types() == FrozenOrderedSet(
[DummyTarget, DummyTarget2])
load_backend(self.bc_builder, backend_package, is_v1_backend=False)
assert self.bc_builder.create().target_types() == FrozenOrderedSet(
[DummyTarget, DummyTarget2])
class PluginTarget(Target):
alias = "plugin_tgt"
core_fields = ()
def plugin_targets():
return [PluginTarget]
self.working_set.add(
self.get_mock_plugin("new-targets",
"0.0.1",
target_types=plugin_targets))
self.load_plugins(["new-targets"], is_v1_plugin=True)
assert self.bc_builder.create().target_types() == FrozenOrderedSet(
[DummyTarget, DummyTarget2, PluginTarget])
self.load_plugins(["new-targets"], is_v1_plugin=False)
assert self.bc_builder.create().target_types() == FrozenOrderedSet(
[DummyTarget, DummyTarget2, PluginTarget])
def test_duplicate_test_mains_different_files(rule_runner: RuleRunner) -> None:
input_digest = rule_runner.make_snapshot(
{
"foo_test.go":
dedent("""
package foo
func TestMain(m *testing.M) {
}
"""),
"bar_test.go":
dedent("""
package foo
func TestMain(m *testing.M) {
}
"""),
}, ).digest
result = rule_runner.request(
GeneratedTestMain,
[
GenerateTestMainRequest(
input_digest,
FrozenOrderedSet(["foo_test.go", "bar_test.go"]),
FrozenOrderedSet(),
"foo",
Address("foo"),
)
],
)
assert result.failed_exit_code_and_stderr is not None
exit_code, stderr = result.failed_exit_code_and_stderr
assert exit_code == 1
assert "multiple definitions of TestMain" in stderr
def test_trie_node_merge_basic() -> None:
one = MutableTrieNode()
one.insert("a/b/c", [Address("1")], recursive=True, first_party=False)
one.insert("a/b/c/d", [Address("2")], recursive=False, first_party=False)
two = MutableTrieNode()
two.insert("a/b/c/d", [Address("3")], recursive=False, first_party=False)
merged = FrozenTrieNode.merge([one.frozen(), two.frozen()])
assert list(merged) == [
(
"a/b/c",
True,
FrozenDict(
{DEFAULT_SYMBOL_NAMESPACE: FrozenOrderedSet([Address("1")])}),
False,
),
(
"a/b/c/d",
False,
FrozenDict({
DEFAULT_SYMBOL_NAMESPACE:
FrozenOrderedSet([Address("2"), Address("3")])
}),
False,
),
]
def test_duplicate_test_mains_different_files(rule_runner: RuleRunner) -> None:
input_digest = rule_runner.make_snapshot(
{
"foo_test.go":
dedent("""
package foo
func TestMain(m *testing.M) {
}
"""),
"bar_test.go":
dedent("""
package foo
func TestMain(m *testing.M) {
}
"""),
}, ).digest
with pytest.raises(ExecutionError) as exc_info:
rule_runner.request(
GeneratedTestMain,
[
GenerateTestMainRequest(
input_digest,
FrozenOrderedSet(["foo_test.go", "bar_test.go"]),
FrozenOrderedSet(),
"foo",
)
],
)
assert "multiple definitions of TestMain" in str(exc_info.value)
async def transitive_targets(targets: Targets) -> TransitiveTargets:
"""Find all the targets transitively depended upon by the target roots.
This uses iteration, rather than recursion, so that we can tolerate dependency cycles. Unlike a
traditional BFS algorithm, we batch each round of traversals via `MultiGet` for improved
performance / concurrency.
"""
visited: OrderedSet[Target] = OrderedSet()
queued = FrozenOrderedSet(targets)
dependency_mapping: Dict[Address, Tuple[Address, ...]] = {}
while queued:
direct_dependencies = await MultiGet(
Get(Targets, DependenciesRequest(tgt.get(Dependencies)))
for tgt in queued)
dependency_mapping.update(
zip(
(t.address for t in queued),
(tuple(t.address for t in deps)
for deps in direct_dependencies),
))
queued = FrozenOrderedSet(
itertools.chain.from_iterable(direct_dependencies)).difference(
visited)
visited.update(queued)
transitive_targets = TransitiveTargets(tuple(targets),
FrozenOrderedSet(visited))
_detect_cycles(tuple(t.address for t in targets), dependency_mapping)
return transitive_targets
def test_lockfile_validation(rule_runner: RuleRunner) -> None:
"""Check that we properly load and validate lockfile metadata for both types of locks.
Note that we don't exhaustively test every source of lockfile failure nor the different options
for `--invalid-lockfile-behavior`, as those are already tested in pex_requirements_test.py.
"""
# We create a lockfile that claims it works with no requirements. It should fail when we try
# to build a PEX with a requirement.
lock_content = PythonLockfileMetadata.new(InterpreterConstraints(),
set()).add_header_to_lockfile(
b"",
regenerate_command="regen",
delimeter="#")
rule_runner.write_files({"lock.txt": lock_content.decode()})
lockfile = Lockfile(
"lock.txt",
file_path_description_of_origin="a test",
resolve_name="a",
req_strings=FrozenOrderedSet("ansicolors"),
)
with engine_error(InvalidLockfileError):
create_pex_and_get_all_data(rule_runner, requirements=lockfile)
lockfile_content = LockfileContent(
FileContent("lock.txt", lock_content),
resolve_name="a",
req_strings=FrozenOrderedSet("ansicolors"),
)
with engine_error(InvalidLockfileError):
create_pex_and_get_all_data(rule_runner, requirements=lockfile_content)
async def find_owners(owners_request: OwnersRequest) -> Owners:
sources_set = FrozenOrderedSet(owners_request.sources)
dirs_set = FrozenOrderedSet(
os.path.dirname(source) for source in sources_set)
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set)
candidate_targets = await Get[HydratedTargets](
AddressSpecs(candidate_specs))
# Match the source globs against the expanded candidate targets.
def owns_any_source(legacy_target: HydratedTarget) -> bool:
"""Given a `HydratedTarget` instance, check if it owns the given source file."""
target_kwargs = legacy_target.adaptor.kwargs()
# Handle `sources`-declaring targets.
# NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target,
# so we don't actually call `fileset.matches` here.
# TODO: This matching logic should be implemented using the rust `fs` crate for two reasons:
# 1) having two implementations isn't great
# 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match
target_sources = target_kwargs.get("sources", None)
return target_sources and any_matches_filespec(
paths=sources_set, spec=target_sources.filespec)
build_file_addresses = await MultiGet(
Get[BuildFileAddress](Address, ht.adaptor.address)
for ht in candidate_targets)
owners = Addresses(
ht.adaptor.address
for ht, bfa in zip(candidate_targets, build_file_addresses)
if LegacyAddressMapper.any_is_declaring_file(bfa, sources_set)
or owns_any_source(ht))
return Owners(owners)
async def get_requirements(
dep_owner: DependencyOwner,
union_membership: UnionMembership,
setup_py_generation: SetupPyGeneration,
) -> ExportedTargetRequirements:
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([dep_owner.exported_target.target.address]),
)
ownable_tgts = [
tgt for tgt in transitive_targets.closure if is_ownable_target(tgt, union_membership)
]
owners = await MultiGet(Get(ExportedTarget, OwnedDependency(tgt)) for tgt in ownable_tgts)
owned_by_us: Set[Target] = set()
owned_by_others: Set[Target] = set()
for tgt, owner in zip(ownable_tgts, owners):
(owned_by_us if owner == dep_owner.exported_target else owned_by_others).add(tgt)
# Get all 3rdparty deps of our owned deps.
#
# Note that we need only consider requirements that are direct dependencies of our owned deps:
# If T depends on R indirectly, then it must be via some direct deps U1, U2, ... For each such U,
# if U is in the owned deps then we'll pick up R through U. And if U is not in the owned deps
# then it's owned by an exported target ET, and so R will be in the requirements for ET, and we
# will require ET.
direct_deps_tgts = await MultiGet(
Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in owned_by_us
)
transitive_excludes: FrozenOrderedSet[Target] = FrozenOrderedSet()
uneval_trans_excl = [
tgt.get(Dependencies).unevaluated_transitive_excludes for tgt in transitive_targets.closure
]
if uneval_trans_excl:
nested_trans_excl = await MultiGet(
Get(Targets, UnparsedAddressInputs, unparsed) for unparsed in uneval_trans_excl
)
transitive_excludes = FrozenOrderedSet(
itertools.chain.from_iterable(excludes for excludes in nested_trans_excl)
)
direct_deps_chained = FrozenOrderedSet(itertools.chain.from_iterable(direct_deps_tgts))
direct_deps_with_excl = direct_deps_chained.difference(transitive_excludes)
reqs = PexRequirements.create_from_requirement_fields(
tgt[PythonRequirementsField]
for tgt in direct_deps_with_excl
if tgt.has_field(PythonRequirementsField)
)
req_strs = list(reqs)
# Add the requirements on any exported targets on which we depend.
kwargs_for_exported_targets_we_depend_on = await MultiGet(
Get(SetupKwargs, OwnedDependency(tgt)) for tgt in owned_by_others
)
req_strs.extend(
f"{kwargs.name}{setup_py_generation.first_party_dependency_version(kwargs.version)}"
for kwargs in set(kwargs_for_exported_targets_we_depend_on)
)
return ExportedTargetRequirements(req_strs)
def from_tool(
cls,
subsystem: PythonToolRequirementsBase,
interpreter_constraints: InterpreterConstraints | None = None,
*,
extra_requirements: Iterable[str] = (),
) -> PythonLockfileRequest:
"""Create a request for a dedicated lockfile for the tool.
If the tool determines its interpreter constraints by using the constraints of user code,
rather than the option `--interpreter-constraints`, you must pass the arg
`interpreter_constraints`.
"""
if not subsystem.uses_lockfile:
return cls(
FrozenOrderedSet(),
InterpreterConstraints(),
resolve_name=subsystem.options_scope,
lockfile_dest=subsystem.lockfile,
)
return cls(
requirements=FrozenOrderedSet((*subsystem.all_requirements, *extra_requirements)),
interpreter_constraints=(
interpreter_constraints
if interpreter_constraints is not None
else subsystem.interpreter_constraints
),
resolve_name=subsystem.options_scope,
lockfile_dest=subsystem.lockfile,
)
async def transitive_targets(
request: TransitiveTargetsRequest) -> TransitiveTargets:
"""Find all the targets transitively depended upon by the target roots."""
dependency_mapping = await Get(_DependencyMapping,
_DependencyMappingRequest(request, True))
# Apply any transitive excludes (`!!` ignores).
transitive_excludes: FrozenOrderedSet[Target] = FrozenOrderedSet()
unevaluated_transitive_excludes = []
for t in (*dependency_mapping.roots_as_targets,
*dependency_mapping.visited):
unparsed = t.get(Dependencies).unevaluated_transitive_excludes
if unparsed.values:
unevaluated_transitive_excludes.append(unparsed)
if unevaluated_transitive_excludes:
nested_transitive_excludes = await MultiGet(
Get(Targets, UnparsedAddressInputs, unparsed)
for unparsed in unevaluated_transitive_excludes)
transitive_excludes = FrozenOrderedSet(
itertools.chain.from_iterable(
excludes for excludes in nested_transitive_excludes))
return TransitiveTargets(
tuple(dependency_mapping.roots_as_targets),
FrozenOrderedSet(
dependency_mapping.visited.difference(transitive_excludes)),
)
def create(cls, rule_entries) -> RuleIndex:
"""Creates a RuleIndex with tasks indexed by their output type."""
rules: OrderedSet[TaskRule] = OrderedSet()
queries: OrderedSet[QueryRule] = OrderedSet()
union_rules: OrderedSet[UnionRule] = OrderedSet()
for entry in rule_entries:
if isinstance(entry, TaskRule):
rules.add(entry)
elif isinstance(entry, UnionRule):
union_rules.add(entry)
elif isinstance(entry, QueryRule):
queries.add(entry)
elif hasattr(entry, "__call__"):
rule = getattr(entry, "rule", None)
if rule is None:
raise TypeError(
f"Expected function {entry} to be decorated with @rule."
)
rules.add(rule)
else:
raise TypeError(
f"Rule entry {entry} had an unexpected type: {type(entry)}. Rules either "
"extend Rule or UnionRule, or are static functions decorated with @rule."
)
return RuleIndex(
rules=FrozenOrderedSet(rules),
queries=FrozenOrderedSet(queries),
union_rules=FrozenOrderedSet(union_rules),
)
async def transitive_targets(
request: TransitiveTargetsRequest) -> TransitiveTargets:
"""Find all the targets transitively depended upon by the target roots.
This uses iteration, rather than recursion, so that we can tolerate dependency cycles. Unlike a
traditional BFS algorithm, we batch each round of traversals via `MultiGet` for improved
performance / concurrency.
"""
roots_as_targets = await Get(Targets, Addresses(request.roots))
visited: OrderedSet[Target] = OrderedSet()
queued = FrozenOrderedSet(roots_as_targets)
dependency_mapping: Dict[Address, Tuple[Address, ...]] = {}
while queued:
direct_dependencies = await MultiGet(
Get(
Targets,
DependenciesRequest(
tgt.get(Dependencies),
include_special_cased_deps=request.
include_special_cased_deps,
),
) for tgt in queued)
dependency_mapping.update(
zip(
(t.address for t in queued),
(tuple(t.address for t in deps)
for deps in direct_dependencies),
))
queued = FrozenOrderedSet(
itertools.chain.from_iterable(direct_dependencies)).difference(
visited)
visited.update(queued)
# NB: We use `roots_as_targets` to get the root addresses, rather than `request.roots`. This
# is because expanding from the `Addresses` -> `Targets` may have resulted in generated
# subtargets being used, so we need to use `roots_as_targets` to have this expansion.
_detect_cycles(tuple(t.address for t in roots_as_targets),
dependency_mapping)
# Apply any transitive excludes (`!!` ignores).
transitive_excludes: FrozenOrderedSet[Target] = FrozenOrderedSet()
unevaluated_transitive_excludes = []
for t in (*roots_as_targets, *visited):
unparsed = t.get(Dependencies).unevaluated_transitive_excludes
if unparsed.values:
unevaluated_transitive_excludes.append(unparsed)
if unevaluated_transitive_excludes:
nested_transitive_excludes = await MultiGet(
Get(Targets, UnparsedAddressInputs, unparsed)
for unparsed in unevaluated_transitive_excludes)
transitive_excludes = FrozenOrderedSet(
itertools.chain.from_iterable(
excludes for excludes in nested_transitive_excludes))
return TransitiveTargets(
tuple(roots_as_targets),
FrozenOrderedSet(visited.difference(transitive_excludes)))
def test_frozen_is_hashable() -> None:
set1 = FrozenOrderedSet("abcabc")
assert hash(set1) == hash(set1.copy())
assert hash(set1) == hash(("a", "b", "c"))
set2 = FrozenOrderedSet("abcd")
assert hash(set1) != hash(set2)
assert hash(set1) != hash(("a", "b", "c", "d"))
def addresses_for_coordinates(
self, coordinates: Iterable[UnversionedCoordinate]
) -> FrozenOrderedSet[Address]:
candidate_artifact_addresses: Set[Address] = set()
for coordinate in coordinates:
candidates = self.artifacts.get(coordinate, FrozenOrderedSet())
candidate_artifact_addresses.update(candidates)
return FrozenOrderedSet(candidate_artifact_addresses)
async def resolve_addresses_from_specs(specs: Specs) -> Addresses:
includes, ignores = await MultiGet(
Get(Addresses, RawSpecs, specs.includes),
Get(Addresses, RawSpecs, specs.ignores),
)
# No matter what, ignores win out over includes. This avoids "specificity wars" and keeps our
# semantics simple/predictable.
return Addresses(FrozenOrderedSet(includes) - FrozenOrderedSet(ignores))
def test_first_party_plugins(rule_runner: RuleRunner) -> None:
rule_runner.write_files(
{
"BUILD": dedent(
"""\
python_requirement(name='flake8', requirements=['flake8==2.11.1'])
python_requirement(name='colors', requirements=['ansicolors'])
"""
),
"flake8-plugins/subdir1/util.py": "",
"flake8-plugins/subdir1/BUILD": dedent(
"""\
python_sources(
interpreter_constraints=['==3.9.*'],
dependencies=['flake8-plugins/subdir2']
)
"""
),
"flake8-plugins/subdir2/another_util.py": "",
"flake8-plugins/subdir2/BUILD": "python_sources(interpreter_constraints=['==3.8.*'])",
"flake8-plugins/plugin.py": "",
"flake8-plugins/BUILD": dedent(
"""\
python_sources(
dependencies=['//:flake8', '//:colors', "flake8-plugins/subdir1"]
)
"""
),
}
)
rule_runner.set_options(
[
"--source-root-patterns=flake8-plugins",
"--flake8-source-plugins=flake8-plugins/plugin.py",
],
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
first_party_plugins = rule_runner.request(Flake8FirstPartyPlugins, [])
assert first_party_plugins.requirement_strings == FrozenOrderedSet(
["ansicolors", "flake8==2.11.1"]
)
assert first_party_plugins.interpreter_constraints_fields == FrozenOrderedSet(
[
InterpreterConstraintsField(ic, Address("", target_name="tgt"))
for ic in (None, ["==3.9.*"], ["==3.8.*"])
]
)
assert (
first_party_plugins.sources_digest
== rule_runner.make_snapshot(
{
f"{Flake8FirstPartyPlugins.PREFIX}/plugin.py": "",
f"{Flake8FirstPartyPlugins.PREFIX}/subdir1/util.py": "",
f"{Flake8FirstPartyPlugins.PREFIX}/subdir2/another_util.py": "",
}
).digest
)
async def find_owners(owners_request: OwnersRequest) -> Owners:
# Determine which of the sources are live and which are deleted.
sources_set_snapshot = await Get(Snapshot,
PathGlobs(owners_request.sources))
live_files = FrozenOrderedSet(sources_set_snapshot.files)
deleted_files = FrozenOrderedSet(s for s in owners_request.sources
if s not in live_files)
live_dirs = FrozenOrderedSet(os.path.dirname(s) for s in live_files)
deleted_dirs = FrozenOrderedSet(os.path.dirname(s) for s in deleted_files)
matching_addresses: OrderedSet[Address] = OrderedSet()
unmatched_sources = set(owners_request.sources)
for live in (True, False):
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
# For live files, we use expanded Targets, which have file level precision but which are
# only created for existing files. For deleted files we use UnexpandedTargets, which have
# the original declared glob.
candidate_targets: Iterable[Target]
if live:
if not live_dirs:
continue
sources_set = live_files
candidate_specs = tuple(
AscendantAddresses(directory=d) for d in live_dirs)
candidate_targets = await Get(Targets,
AddressSpecs(candidate_specs))
else:
if not deleted_dirs:
continue
sources_set = deleted_files
candidate_specs = tuple(
AscendantAddresses(directory=d) for d in deleted_dirs)
candidate_targets = await Get(UnexpandedTargets,
AddressSpecs(candidate_specs))
build_file_addresses = await MultiGet(
Get(BuildFileAddress, Address, tgt.address)
for tgt in candidate_targets)
for candidate_tgt, bfa in zip(candidate_targets, build_file_addresses):
matching_files = set(
matches_filespec(candidate_tgt.get(Sources).filespec,
paths=sources_set))
if not matching_files and bfa.rel_path not in sources_set:
continue
unmatched_sources -= matching_files
matching_addresses.add(candidate_tgt.address)
if (unmatched_sources and owners_request.owners_not_found_behavior !=
OwnersNotFoundBehavior.ignore):
_log_or_raise_unmatched_owners(
[PurePath(path) for path in unmatched_sources],
owners_request.owners_not_found_behavior)
return Owners(matching_addresses)
def test_first_party_plugins(rule_runner: RuleRunner) -> None:
rule_runner.write_files(
{
"BUILD": dedent(
"""\
python_requirement_library(name='pylint', requirements=['pylint==2.6.2'])
python_requirement_library(name='colors', requirements=['ansicolors'])
"""
),
"pylint-plugins/subdir1/util.py": "",
"pylint-plugins/subdir1/BUILD": dedent(
"""\
python_library(
interpreter_constraints=['==3.5.*'],
dependencies=['pylint-plugins/subdir2']
)
"""
),
"pylint-plugins/subdir2/another_util.py": "",
"pylint-plugins/subdir2/BUILD": ("python_library(interpreter_constraints=['==3.4.*'])"),
"pylint-plugins/plugin.py": "",
"pylint-plugins/BUILD": dedent(
"""\
python_library(
dependencies=['//:pylint', '//:colors', "pylint-plugins/subdir1"]
)
"""
),
}
)
rule_runner.set_options(
[
"--source-root-patterns=pylint-plugins",
"--pylint-source-plugins=pylint-plugins/plugin.py",
]
)
first_party_plugins = rule_runner.request(PylintFirstPartyPlugins, [])
assert first_party_plugins.requirement_strings == FrozenOrderedSet(
["ansicolors", "pylint==2.6.2"]
)
assert first_party_plugins.interpreter_constraints_fields == FrozenOrderedSet(
[
InterpreterConstraintsField(ic, Address("", target_name="tgt"))
for ic in (None, ["==3.5.*"], ["==3.4.*"])
]
)
assert (
first_party_plugins.sources_digest
== rule_runner.make_snapshot(
{
f"{PylintFirstPartyPlugins.PREFIX}/plugin.py": "",
f"{PylintFirstPartyPlugins.PREFIX}/subdir1/util.py": "",
f"{PylintFirstPartyPlugins.PREFIX}/subdir2/another_util.py": "",
}
).digest
)
def get_disambiguated(
*,
ambiguous: List[Address],
ignores: List[Address],
includes: Optional[List[Address]] = None,
) -> Optional[Address]:
epd = ExplicitlyProvidedDependencies(
includes=FrozenOrderedSet(includes or []), ignores=FrozenOrderedSet(ignores)
)
return epd.disambiguated_via_ignores(tuple(ambiguous))
async def mypy_typecheck(
request: MyPyRequest, mypy: MyPy, python_setup: PythonSetup
) -> TypecheckResults:
if mypy.skip:
return TypecheckResults([], typechecker_name="MyPy")
# We batch targets by their interpreter constraints to ensure, for example, that all Python 2
# targets run together and all Python 3 targets run together. We can only do this by setting
# the `--python-version` option, but we allow the user to set it as a safety valve. We warn if
# they've set the option.
config_files = await Get(ConfigFiles, ConfigFilesRequest, mypy.config_request)
config_content = await Get(DigestContents, Digest, config_files.snapshot.digest)
python_version_configured = check_and_warn_if_python_version_configured(
config=next(iter(config_content), None), args=mypy.args
)
# When determining how to batch by interpreter constraints, we must consider the entire
# transitive closure to get the final resulting constraints.
# TODO(#10863): Improve the performance of this.
transitive_targets_per_field_set = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address]))
for field_set in request.field_sets
)
interpreter_constraints_to_transitive_targets = defaultdict(set)
for transitive_targets in transitive_targets_per_field_set:
interpreter_constraints = PexInterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup
) or PexInterpreterConstraints(mypy.interpreter_constraints)
interpreter_constraints_to_transitive_targets[interpreter_constraints].add(
transitive_targets
)
partitions = []
for interpreter_constraints, all_transitive_targets in sorted(
interpreter_constraints_to_transitive_targets.items()
):
combined_roots: OrderedSet[Target] = OrderedSet()
combined_closure: OrderedSet[Target] = OrderedSet()
for transitive_targets in all_transitive_targets:
combined_roots.update(transitive_targets.roots)
combined_closure.update(transitive_targets.closure)
partitions.append(
MyPyPartition(
FrozenOrderedSet(combined_roots),
FrozenOrderedSet(combined_closure),
interpreter_constraints,
python_version_already_configured=python_version_configured,
)
)
partitioned_results = await MultiGet(
Get(TypecheckResult, MyPyPartition, partition) for partition in partitions
)
return TypecheckResults(partitioned_results, typechecker_name="MyPy")
async def transitive_dependency_mapping(
request: _DependencyMappingRequest) -> _DependencyMapping:
"""This uses iteration, rather than recursion, so that we can tolerate dependency cycles.
Unlike a traditional BFS algorithm, we batch each round of traversals via `MultiGet` for
improved performance / concurrency.
"""
roots_as_targets = await Get(UnexpandedTargets,
Addresses(request.tt_request.roots))
visited: OrderedSet[Target] = OrderedSet()
queued = FrozenOrderedSet(roots_as_targets)
dependency_mapping: dict[Address, tuple[Address, ...]] = {}
while queued:
direct_dependencies: tuple[Collection[Target], ...]
if request.expanded_targets:
direct_dependencies = await MultiGet(
Get(
Targets,
DependenciesRequest(
tgt.get(Dependencies),
include_special_cased_deps=request.tt_request.
include_special_cased_deps,
),
) for tgt in queued)
else:
direct_dependencies = await MultiGet(
Get(
UnexpandedTargets,
DependenciesRequest(
tgt.get(Dependencies),
include_special_cased_deps=request.tt_request.
include_special_cased_deps,
),
) for tgt in queued)
dependency_mapping.update(
zip(
(t.address for t in queued),
(tuple(t.address for t in deps)
for deps in direct_dependencies),
))
queued = FrozenOrderedSet(
itertools.chain.from_iterable(direct_dependencies)).difference(
visited)
visited.update(queued)
# NB: We use `roots_as_targets` to get the root addresses, rather than `request.roots`. This
# is because expanding from the `Addresses` -> `Targets` may have resulted in generated
# targets being used, so we need to use `roots_as_targets` to have this expansion.
# TODO(#12871): Fix this to not be based on generated targets.
_detect_cycles(tuple(t.address for t in roots_as_targets),
dependency_mapping)
return _DependencyMapping(FrozenDict(dependency_mapping),
FrozenOrderedSet(visited), roots_as_targets)
def enumerate_python_versions(
self, interpreter_universe: Iterable[str]
) -> FrozenOrderedSet[tuple[int, int, int]]:
"""Return a set of all plausible (major, minor, patch) tuples for all Python 2.7/3.x in the
specified interpreter universe that matches this set of interpreter constraints.
This also validates our assumptions around the `interpreter_universe`:
- Python 2.7 is the only Python 2 version in the universe, if at all.
- Python 3 is the last major release of Python, which the core devs have committed to in
public several times.
"""
if not self:
return FrozenOrderedSet()
minors = []
for major_minor in interpreter_universe:
major, minor = _major_minor_to_int(major_minor)
if major == 2:
if minor != 7:
raise AssertionError(
"Unexpected value in `[python-setup].interpreter_versions_universe`: "
f"{major_minor}. Expected the only Python 2 value to be '2.7', given that "
f"all other versions are unmaintained or do not exist."
)
minors.append((2, minor))
elif major == 3:
minors.append((3, minor))
else:
raise AssertionError(
"Unexpected value in `[python-setup].interpreter_versions_universe`: "
f"{major_minor}. Expected to only include '2.7' and/or Python 3 versions, "
"given that Python 3 will be the last major Python version. Please open an "
"issue at https://github.com/pantsbuild/pants/issues/new if this is no longer "
"true."
)
valid_patches = FrozenOrderedSet(
(major, minor, patch)
for (major, minor) in sorted(minors)
for patch in self._valid_patch_versions(major, minor)
)
if not valid_patches:
raise ValueError(
f"The interpreter constraints `{self}` are not compatible with any of the "
"interpreter versions from `[python-setup].interpreter_versions_universe`.\n\n"
"Please either change these interpreter constraints or update the "
"`interpreter_versions_universe` to include the interpreters set in these "
"constraints. Run `./pants help-advanced python-setup` for more information on the "
"`interpreter_versions_universe` option."
)
return valid_patches
def test_multiple_resolves() -> None:
rule_runner = RuleRunner(
rules=[
setup_user_lockfile_requests,
SubsystemRule(PythonSetup),
QueryRule(UserGenerateLockfiles,
[RequestedPythonUserResolveNames]),
],
target_types=[PythonRequirementTarget],
)
rule_runner.write_files({
"BUILD":
dedent("""\
python_requirement(
name='a',
requirements=['a'],
resolve='a',
)
python_requirement(
name='b',
requirements=['b'],
resolve='b',
)
"""),
})
rule_runner.set_options(
[
"--python-resolves={'a': 'a.lock', 'b': 'b.lock'}",
# Override interpreter constraints for 'b', but use default for 'a'.
"--python-resolves-to-interpreter-constraints={'b': ['==3.7.*']}",
"--python-enable-resolves",
"--python-lockfile-generator=pex",
],
env_inherit=PYTHON_BOOTSTRAP_ENV,
)
result = rule_runner.request(UserGenerateLockfiles,
[RequestedPythonUserResolveNames(["a", "b"])])
assert set(result) == {
GeneratePythonLockfile(
requirements=FrozenOrderedSet(["a"]),
interpreter_constraints=InterpreterConstraints(
PythonSetup.default_interpreter_constraints),
resolve_name="a",
lockfile_dest="a.lock",
use_pex=True,
),
GeneratePythonLockfile(
requirements=FrozenOrderedSet(["b"]),
interpreter_constraints=InterpreterConstraints(["==3.7.*"]),
resolve_name="b",
lockfile_dest="b.lock",
use_pex=True,
),
}
def create(self) -> BuildConfiguration:
registered_aliases = BuildFileAliases(
objects=self._exposed_object_by_alias.copy(),
context_aware_object_factories=self._exposed_context_aware_object_factory_by_alias.copy(),
)
return BuildConfiguration(
registered_aliases=registered_aliases,
optionables=FrozenOrderedSet(self._optionables),
rules=FrozenOrderedSet(self._rules),
union_rules=FrozenOrderedSet(self._union_rules),
target_types=FrozenOrderedSet(self._target_types),
)
def maybe_warn(
*,
ambiguous: List[Address],
ignores: Optional[List[Address]] = None,
includes: Optional[List[Address]] = None,
) -> None:
caplog.clear()
epd = ExplicitlyProvidedDependencies(
includes=FrozenOrderedSet(includes or []), ignores=FrozenOrderedSet(ignores or [])
)
epd.maybe_warn_of_ambiguous_dependency_inference(
tuple(ambiguous), Address("some_dir"), import_reference="file", context="foo"
)
def test_explicitly_provided_dependencies_remaining_after_disambiguation(
) -> None:
# First check disambiguation via ignores (`!` and `!!`).
addr = Address("", target_name="a")
generated_addr = Address("", target_name="b", generated_name="gen")
epd = ExplicitlyProvidedDependencies(
Address("", target_name="input_tgt"),
includes=FrozenOrderedSet(),
ignores=FrozenOrderedSet([addr, generated_addr]),
)
def assert_disambiguated_via_ignores(ambiguous: List[Address],
expected: Set[Address]) -> None:
assert (epd.remaining_after_disambiguation(
tuple(ambiguous), owners_must_be_ancestors=False) == expected)
assert_disambiguated_via_ignores([], set())
assert_disambiguated_via_ignores([addr], set())
assert_disambiguated_via_ignores([generated_addr], set())
assert_disambiguated_via_ignores([addr, generated_addr], set())
# Generated targets are covered if their original target generator is in the ignores.
assert_disambiguated_via_ignores(
[Address("", target_name="a", generated_name="gen")], set())
bad_tgt = Address("", target_name="x")
bad_generated_tgt = Address("", target_name="x", generated_name="gen")
assert_disambiguated_via_ignores([bad_tgt], {bad_tgt})
assert_disambiguated_via_ignores([bad_generated_tgt], {bad_generated_tgt})
assert_disambiguated_via_ignores([bad_generated_tgt, addr, generated_addr],
{bad_generated_tgt})
# Check disambiguation via `owners_must_be_ancestors`.
epd = ExplicitlyProvidedDependencies(Address("src/lang/project"),
FrozenOrderedSet(),
FrozenOrderedSet())
valid_candidates = {
Address("src/lang/project", target_name="another_tgt"),
Address("src/lang"),
Address("src"),
Address("", target_name="root_owner"),
}
invalid_candidates = {
Address("tests/lang"),
Address("src/another_lang"),
Address("src/lang/another_project"),
Address("src/lang/project/subdir"),
}
assert (epd.remaining_after_disambiguation(
(*valid_candidates, *invalid_candidates),
owners_must_be_ancestors=True) == valid_candidates)
def find_python_imports(source_code: str, *, module_name: str) -> ParsedPythonImports:
parse_result = parse_file(source_code)
# If there were syntax errors, gracefully early return. This is more user friendly than
# propagating the exception. Dependency inference simply won't be used for that file, and
# it'll be up to the tool actually being run (e.g. Pytest or Flake8) to error.
if parse_result is None:
return ParsedPythonImports(FrozenOrderedSet(), FrozenOrderedSet())
tree, ast_visitor_cls = parse_result
ast_visitor = ast_visitor_cls(module_name)
ast_visitor.visit(tree)
return ParsedPythonImports(
explicit_imports=FrozenOrderedSet(sorted(ast_visitor.explicit_imports)),
inferred_imports=FrozenOrderedSet(sorted(ast_visitor.inferred_imports)),
)
async def transitive_targets_lite(request: TransitiveTargetsRequestLite) -> TransitiveTargets:
roots_as_targets = await Get(Targets, Addresses(request.roots))
visited: OrderedSet[Target] = OrderedSet()
queued = FrozenOrderedSet(roots_as_targets)
dependency_mapping: Dict[Address, Tuple[Address, ...]] = {}
while queued:
direct_dependencies_addresses_per_tgt = await MultiGet(
Get(Addresses, DependenciesRequestLite(tgt.get(Dependencies))) for tgt in queued
)
direct_dependencies_per_tgt = []
for addresses_per_tgt in direct_dependencies_addresses_per_tgt:
wrapped_tgts = await MultiGet(
Get(WrappedTarget, Address, addr) for addr in addresses_per_tgt
)
direct_dependencies_per_tgt.append(
tuple(wrapped_t.target for wrapped_t in wrapped_tgts)
)
dependency_mapping.update(
zip(
(t.address for t in queued),
(tuple(t.address for t in deps) for deps in direct_dependencies_per_tgt),
)
)
queued = FrozenOrderedSet(
itertools.chain.from_iterable(direct_dependencies_per_tgt)
).difference(visited)
visited.update(queued)
# NB: We use `roots_as_targets` to get the root addresses, rather than `request.roots`. This
# is because expanding from the `Addresses` -> `Targets` may have resulted in generated
# subtargets being used, so we need to use `roots_as_targets` to have this expansion.
_detect_cycles(tuple(t.address for t in roots_as_targets), dependency_mapping)
# Apply any transitive excludes (`!!` ignores).
wrapped_transitive_excludes = await MultiGet(
Get(
WrappedTarget, AddressInput, AddressInput.parse(addr, relative_to=tgt.address.spec_path)
)
for tgt in (*roots_as_targets, *visited)
for addr in tgt.get(Dependencies).unevaluated_transitive_excludes.values
)
transitive_excludes = FrozenOrderedSet(
wrapped_t.target for wrapped_t in wrapped_transitive_excludes
)
return TransitiveTargets(
tuple(roots_as_targets), FrozenOrderedSet(visited.difference(transitive_excludes))
)
