async def setup_ipython_lockfile(
_: IPythonLockfileSentinel, ipython: IPython,
python_setup: PythonSetup) -> GeneratePythonLockfile:
if not ipython.uses_lockfile:
return GeneratePythonLockfile.from_tool(
ipython, use_pex=python_setup.generate_lockfiles_with_pex)
# IPython is often run against the whole repo (`./pants repl ::`), but it is possible to run
# on subsets of the codebase with disjoint interpreter constraints, such as
# `./pants repl py2::` and then `./pants repl py3::`. Still, even with those subsets possible,
# we need a single lockfile that works with all possible Python interpreters in use.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
unique_constraints = {
InterpreterConstraints.create_from_compatibility_fields(
[tgt[InterpreterConstraintsField]], python_setup)
for tgt in all_tgts if tgt.has_field(InterpreterConstraintsField)
}
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
ipython,
constraints
or InterpreterConstraints(python_setup.interpreter_constraints),
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_pytest_lockfile(
_: PytestLockfileSentinel, pytest: PyTest,
python_setup: PythonSetup) -> PythonLockfileRequest:
if not pytest.uses_lockfile:
return PythonLockfileRequest.from_tool(pytest)
# Even though we run each python_tests target in isolation, we need a single lockfile that
# works with them all (and their transitive deps).
#
# This first computes the constraints for each individual `python_test` target
# (which will AND across each target in the closure). Then, it ORs all unique resulting
# interpreter constraints. The net effect is that every possible Python interpreter used will
# be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
transitive_targets_per_test = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_tgts if PythonTestFieldSet.is_applicable(tgt))
unique_constraints = {
InterpreterConstraints.create_from_targets(transitive_targets.closure,
python_setup)
for transitive_targets in transitive_targets_per_test
}
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return PythonLockfileRequest.from_tool(
pytest, constraints
or InterpreterConstraints(python_setup.interpreter_constraints))
async def setup_setuptools_lockfile(
_: SetuptoolsLockfileSentinel, setuptools: Setuptools, python_setup: PythonSetup
) -> GeneratePythonLockfile:
if not setuptools.uses_custom_lockfile:
return GeneratePythonLockfile.from_tool(
setuptools, use_pex=python_setup.generate_lockfiles_with_pex
)
all_tgts = await Get(AllTargets, AllTargetsRequest())
transitive_targets_per_python_dist = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_tgts
if PythonDistributionFieldSet.is_applicable(tgt)
)
unique_constraints = {
InterpreterConstraints.create_from_targets(transitive_targets.closure, python_setup)
or InterpreterConstraints(python_setup.interpreter_constraints)
for transitive_targets in transitive_targets_per_python_dist
}
constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
setuptools,
constraints or InterpreterConstraints(python_setup.interpreter_constraints),
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_mypy_lockfile(
_: MyPyLockfileSentinel,
first_party_plugins: MyPyFirstPartyPlugins,
mypy: MyPy,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
if not mypy.uses_lockfile:
return GeneratePythonLockfile.from_tool(
mypy, use_pex=python_setup.generate_lockfiles_with_pex)
constraints = mypy.interpreter_constraints
if mypy.options.is_default("interpreter_constraints"):
all_tgts = await Get(AllTargets, AllTargetsRequest())
all_transitive_targets = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_tgts if MyPyFieldSet.is_applicable(tgt))
unique_constraints = {
InterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup)
for transitive_targets in all_transitive_targets
}
code_constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
if code_constraints.requires_python38_or_newer(
python_setup.interpreter_universe):
constraints = code_constraints
return GeneratePythonLockfile.from_tool(
mypy,
constraints,
extra_requirements=first_party_plugins.requirement_strings,
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_bandit_lockfile(
_: BanditLockfileSentinel, bandit: Bandit, python_setup: PythonSetup
) -> GeneratePythonLockfile:
if not bandit.uses_lockfile:
return GeneratePythonLockfile.from_tool(
bandit, use_pex=python_setup.generate_lockfiles_with_pex
)
# While Bandit will run in partitions, we need a single lockfile that works with every
# partition.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
unique_constraints = {
InterpreterConstraints.create_from_targets([tgt], python_setup)
for tgt in all_tgts
if BanditFieldSet.is_applicable(tgt)
}
constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
bandit,
constraints or InterpreterConstraints(python_setup.interpreter_constraints),
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def _pylint_interpreter_constraints(
first_party_plugins: PylintFirstPartyPlugins,
python_setup: PythonSetup,
) -> InterpreterConstraints:
# While Pylint will run in partitions, we need a set of constraints that works with every
# partition. We must also consider any 3rd-party requirements used by 1st-party plugins.
#
# This first computes the constraints for each individual target. Then, it ORs all unique
# resulting interpreter constraints. The net effect is that every possible Python interpreter
# used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
unique_constraints = {
InterpreterConstraints.create_from_compatibility_fields(
(
tgt[InterpreterConstraintsField],
*first_party_plugins.interpreter_constraints_fields,
),
python_setup,
)
for tgt in all_tgts
if PylintFieldSet.is_applicable(tgt)
}
if not unique_constraints:
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
first_party_plugins.interpreter_constraints_fields,
python_setup,
)
)
constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints))
return constraints or InterpreterConstraints(python_setup.interpreter_constraints)
async def _flake8_interpreter_constraints(
first_party_plugins: Flake8FirstPartyPlugins,
python_setup: PythonSetup,
) -> InterpreterConstraints:
# While Flake8 will run in partitions, we need a set of constraints that works with every
# partition.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
unique_constraints = {
InterpreterConstraints.create_from_compatibility_fields(
(
tgt[InterpreterConstraintsField],
*first_party_plugins.interpreter_constraints_fields,
),
python_setup,
)
for tgt in all_tgts if Flake8FieldSet.is_applicable(tgt)
}
if not unique_constraints:
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
first_party_plugins.interpreter_constraints_fields,
python_setup,
))
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return constraints or InterpreterConstraints(
python_setup.interpreter_constraints)
async def setup_pylint_lockfile(
_: PylintLockfileSentinel,
first_party_plugins: PylintFirstPartyPlugins,
pylint: Pylint,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
if not pylint.uses_lockfile:
return GeneratePythonLockfile.from_tool(
pylint, use_pex=python_setup.generate_lockfiles_with_pex)
# While Pylint will run in partitions, we need a single lockfile that works with every
# partition. We must also consider any 3rd-party requirements used by 1st-party plugins.
#
# This first computes the constraints for each individual target, including its direct
# dependencies (which will AND across each target in the closure). Then, it ORs all unique
# resulting interpreter constraints. The net effect is that every possible Python interpreter
# used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
relevant_targets = tuple(tgt for tgt in all_tgts
if PylintFieldSet.is_applicable(tgt))
direct_deps_per_target = await MultiGet(
Get(Targets, DependenciesRequest(tgt.get(Dependencies)))
for tgt in relevant_targets)
unique_constraints = set()
for tgt, direct_deps in zip(relevant_targets, direct_deps_per_target):
constraints_fields = (t[InterpreterConstraintsField]
for t in (tgt, *direct_deps)
if t.has_field(InterpreterConstraintsField))
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
(*constraints_fields,
*first_party_plugins.interpreter_constraints_fields),
python_setup,
))
if not unique_constraints:
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
first_party_plugins.interpreter_constraints_fields,
python_setup,
))
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
pylint,
constraints
or InterpreterConstraints(python_setup.interpreter_constraints),
extra_requirements=first_party_plugins.requirement_strings,
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def _black_interpreter_constraints(
black: Black, python_setup: PythonSetup
) -> InterpreterConstraints:
constraints = black.interpreter_constraints
if black.options.is_default("interpreter_constraints"):
all_tgts = await Get(AllTargets, AllTargetsRequest())
# TODO: fix to use `FieldSet.is_applicable()`.
code_constraints = InterpreterConstraints.create_from_targets(
(tgt for tgt in all_tgts if not tgt.get(SkipBlackField).value), python_setup
)
if code_constraints is not None and code_constraints.requires_python38_or_newer(
python_setup.interpreter_universe
):
constraints = code_constraints
return constraints
async def _bandit_interpreter_constraints(
python_setup: PythonSetup) -> InterpreterConstraints:
# While Bandit will run in partitions, we need a set of constraints that works with every
# partition.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
unique_constraints = {
InterpreterConstraints.create_from_targets([tgt], python_setup)
for tgt in all_tgts if BanditFieldSet.is_applicable(tgt)
}
constraints = InterpreterConstraints(
itertools.chain.from_iterable(ic for ic in unique_constraints if ic))
return constraints or InterpreterConstraints(
python_setup.interpreter_constraints)
async def _mypy_interpreter_constraints(
mypy: MyPy, python_setup: PythonSetup) -> InterpreterConstraints:
constraints = mypy.interpreter_constraints
if mypy.options.is_default("interpreter_constraints"):
all_tgts = await Get(AllTargets, AllTargetsRequest())
unique_constraints = {
InterpreterConstraints.create_from_targets([tgt], python_setup)
for tgt in all_tgts if MyPyFieldSet.is_applicable(tgt)
}
code_constraints = InterpreterConstraints(
itertools.chain.from_iterable(ic for ic in unique_constraints
if ic))
if code_constraints.requires_python38_or_newer(
python_setup.interpreter_universe):
constraints = code_constraints
return constraints
async def setup_black_lockfile(
_: BlackLockfileSentinel, black: Black,
python_setup: PythonSetup) -> PythonLockfileRequest:
if not black.uses_lockfile:
return PythonLockfileRequest.from_tool(black)
constraints = black.interpreter_constraints
if black.options.is_default("interpreter_constraints"):
all_tgts = await Get(AllTargets, AllTargetsRequest())
# TODO: fix to use `FieldSet.is_applicable()`.
code_constraints = InterpreterConstraints.create_from_targets(
(tgt for tgt in all_tgts if not tgt.get(SkipBlackField).value),
python_setup)
if code_constraints.requires_python38_or_newer(
python_setup.interpreter_universe):
constraints = code_constraints
return PythonLockfileRequest.from_tool(black, constraints)
async def setup_flake8_lockfile(
_: Flake8LockfileSentinel,
first_party_plugins: Flake8FirstPartyPlugins,
flake8: Flake8,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
if not flake8.uses_lockfile:
return GeneratePythonLockfile.from_tool(
flake8, use_pex=python_setup.generate_lockfiles_with_pex
)
# While Flake8 will run in partitions, we need a single lockfile that works with every
# partition.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
relevant_targets = tuple(tgt for tgt in all_tgts if Flake8FieldSet.is_applicable(tgt))
unique_constraints = set()
for tgt in relevant_targets:
if tgt.has_field(InterpreterConstraintsField):
constraints_field = tgt[InterpreterConstraintsField]
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
(constraints_field, *first_party_plugins.interpreter_constraints_fields),
python_setup,
)
)
if not unique_constraints:
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
first_party_plugins.interpreter_constraints_fields,
python_setup,
)
)
constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
flake8,
constraints or InterpreterConstraints(python_setup.interpreter_constraints),
extra_requirements=first_party_plugins.requirement_strings,
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_flake8_lockfile(
_: Flake8LockfileSentinel, flake8: Flake8,
python_setup: PythonSetup) -> PythonLockfileRequest:
if not flake8.uses_lockfile:
return PythonLockfileRequest.from_tool(flake8)
# While Flake8 will run in partitions, we need a single lockfile that works with every
# partition.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
unique_constraints = {
InterpreterConstraints.create_from_targets([tgt], python_setup)
for tgt in all_tgts if Flake8FieldSet.is_applicable(tgt)
}
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return PythonLockfileRequest.from_tool(
flake8, constraints
or InterpreterConstraints(python_setup.interpreter_constraints))
async def find_all_unexpanded_targets_singleton() -> AllUnexpandedTargets:
return await Get(AllUnexpandedTargets, AllTargetsRequest())
async def py_constraints(
addresses: Addresses,
console: Console,
py_constraints_subsystem: PyConstraintsSubsystem,
python_setup: PythonSetup,
registered_target_types: RegisteredTargetTypes,
union_membership: UnionMembership,
) -> PyConstraintsGoal:
if py_constraints_subsystem.summary:
if addresses:
console.print_stderr(
"The `py-constraints --summary` goal does not take file/target arguments. Run "
"`help py-constraints` for more details.")
return PyConstraintsGoal(exit_code=1)
all_targets = await Get(AllTargets, AllTargetsRequest())
all_python_targets = tuple(t for t in all_targets
if t.has_field(InterpreterConstraintsField))
constraints_per_tgt = [
InterpreterConstraints.create_from_targets([tgt], python_setup)
for tgt in all_python_targets
]
transitive_targets_per_tgt = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_python_targets)
transitive_constraints_per_tgt = [
InterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup)
for transitive_targets in transitive_targets_per_tgt
]
dependees_per_root = await MultiGet(
Get(
Dependees,
DependeesRequest(
[tgt.address], transitive=True, include_roots=False))
for tgt in all_python_targets)
data = [{
"Target": tgt.address.spec,
"Constraints": str(constraints),
"Transitive Constraints": str(transitive_constraints),
"# Dependencies": len(transitive_targets.dependencies),
"# Dependees": len(dependees),
} for tgt, constraints, transitive_constraints, transitive_targets,
dependees in zip(
all_python_targets,
constraints_per_tgt,
transitive_constraints_per_tgt,
transitive_targets_per_tgt,
dependees_per_root,
)]
with py_constraints_subsystem.output_sink(console) as stdout:
writer = csv.DictWriter(
stdout,
fieldnames=[
"Target",
"Constraints",
"Transitive Constraints",
"# Dependencies",
"# Dependees",
],
)
writer.writeheader()
for entry in data:
writer.writerow(entry)
return PyConstraintsGoal(exit_code=0)
transitive_targets = await Get(TransitiveTargets,
TransitiveTargetsRequest(addresses))
final_constraints = InterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup)
if not final_constraints:
target_types_with_constraints = sorted(
tgt_type.alias for tgt_type in registered_target_types.types
if tgt_type.class_has_field(InterpreterConstraintsField,
union_membership))
logger.warning(
"No Python files/targets matched for the `py-constraints` goal. All target types with "
f"Python interpreter constraints: {', '.join(target_types_with_constraints)}"
)
return PyConstraintsGoal(exit_code=0)
constraints_to_addresses = defaultdict(set)
for tgt in transitive_targets.closure:
constraints = InterpreterConstraints.create_from_targets([tgt],
python_setup)
if not constraints:
continue
constraints_to_addresses[constraints].add(tgt.address)
with py_constraints_subsystem.output(console) as output_stdout:
output_stdout(f"Final merged constraints: {final_constraints}\n")
if len(addresses) > 1:
merged_constraints_warning = (
"(These are the constraints used if you were to depend on all of the input "
"files/targets together, even though they may end up never being used together in "
"the real world. Consider using a more precise query or running "
f"`{bin_name()} py-constraints --summary`.)\n")
output_stdout(indent(fill(merged_constraints_warning, 80), " "))
for constraint, addrs in sorted(constraints_to_addresses.items()):
output_stdout(f"\n{constraint}\n")
for addr in sorted(addrs):
output_stdout(f" {addr}\n")
return PyConstraintsGoal(exit_code=0)
