async def setup_mypy_lockfile(
_: MyPyLockfileSentinel,
first_party_plugins: MyPyFirstPartyPlugins,
mypy: MyPy,
python_setup: PythonSetup,
) -> PythonLockfileRequest:
if not mypy.uses_lockfile:
return PythonLockfileRequest.from_tool(mypy)
constraints = mypy.interpreter_constraints
if mypy.options.is_default("interpreter_constraints"):
all_build_targets = await Get(UnexpandedTargets,
AddressSpecs([DescendantAddresses("")]))
all_transitive_targets = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_build_targets if MyPyFieldSet.is_applicable(tgt))
unique_constraints = {
InterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup)
for transitive_targets in all_transitive_targets
}
code_constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
if code_constraints.requires_python38_or_newer(
python_setup.interpreter_universe):
constraints = code_constraints
return PythonLockfileRequest.from_tool(
mypy,
constraints,
extra_requirements=first_party_plugins.requirement_strings)
async def setup_pytest_lockfile(
_: PytestLockfileSentinel, pytest: PyTest,
python_setup: PythonSetup) -> PythonLockfileRequest:
if not pytest.uses_lockfile:
return PythonLockfileRequest.from_tool(pytest)
# Even though we run each python_tests target in isolation, we need a single lockfile that
# works with them all (and their transitive deps).
#
# This first computes the constraints for each individual `python_tests` target
# (which will AND across each target in the closure). Then, it ORs all unique resulting
# interpreter constraints. The net effect is that every possible Python interpreter used will
# be covered.
all_build_targets = await Get(UnexpandedTargets,
AddressSpecs([DescendantAddresses("")]))
transitive_targets_per_test = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_build_targets if PythonTestFieldSet.is_applicable(tgt))
unique_constraints = {
InterpreterConstraints.create_from_targets(transitive_targets.closure,
python_setup)
for transitive_targets in transitive_targets_per_test
}
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return PythonLockfileRequest.from_tool(
pytest, constraints
or InterpreterConstraints(python_setup.interpreter_constraints))
async def setup_ipython_lockfile(
_: IPythonLockfileSentinel, ipython: IPython, python_setup: PythonSetup
) -> PythonLockfileRequest:
if not ipython.uses_lockfile:
return PythonLockfileRequest.from_tool(ipython)
# IPython is often run against the whole repo (`./pants repl ::`), but it is possible to run
# on subsets of the codebase with disjoint interpreter constraints, such as
# `./pants repl py2::` and then `./pants repl py3::`. Still, even with those subsets possible,
# we need a single lockfile that works with all possible Python interpreters in use.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_build_targets = await Get(UnexpandedTargets, AddressSpecs([DescendantAddresses("")]))
unique_constraints = {
InterpreterConstraints.create_from_compatibility_fields(
[tgt[InterpreterConstraintsField]], python_setup
)
for tgt in all_build_targets
if tgt.has_field(InterpreterConstraintsField)
}
constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints))
return PythonLockfileRequest.from_tool(
ipython, constraints or InterpreterConstraints(python_setup.interpreter_constraints)
)
async def setup_pylint_lockfile(
_: PylintLockfileSentinel,
first_party_plugins: PylintFirstPartyPlugins,
pylint: Pylint,
python_setup: PythonSetup,
) -> PythonLockfileRequest:
if not pylint.uses_lockfile:
return PythonLockfileRequest.from_tool(pylint)
# While Pylint will run in partitions, we need a single lockfile that works with every
# partition. We must also consider any 3rd-party requirements used by 1st-party plugins.
#
# This first computes the constraints for each individual target, including its direct
# dependencies (which will AND across each target in the closure). Then, it ORs all unique
# resulting interpreter constraints. The net effect is that every possible Python interpreter
# used will be covered.
all_build_targets = await Get(UnexpandedTargets,
AddressSpecs([DescendantAddresses("")]))
relevant_targets = tuple(tgt for tgt in all_build_targets
if PylintFieldSet.is_applicable(tgt))
direct_deps_per_target = await MultiGet(
Get(UnexpandedTargets, DependenciesRequest(tgt.get(Dependencies)))
for tgt in relevant_targets)
unique_constraints = set()
for tgt, direct_deps in zip(relevant_targets, direct_deps_per_target):
constraints_fields = (t[InterpreterConstraintsField]
for t in (tgt, *direct_deps)
if t.has_field(InterpreterConstraintsField))
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
(*constraints_fields,
*first_party_plugins.interpreter_constraints_fields),
python_setup,
))
if not unique_constraints:
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
first_party_plugins.interpreter_constraints_fields,
python_setup,
))
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return PythonLockfileRequest.from_tool(
pylint,
constraints
or InterpreterConstraints(python_setup.interpreter_constraints),
extra_requirements=first_party_plugins.requirement_strings,
)
async def setup_black_lockfile(
_: BlackLockfileSentinel, black: Black,
python_setup: PythonSetup) -> PythonLockfileRequest:
if not black.uses_lockfile:
return PythonLockfileRequest.from_tool(black)
constraints = black.interpreter_constraints
if black.options.is_default("interpreter_constraints"):
all_tgts = await Get(AllTargets, AllTargetsRequest())
# TODO: fix to use `FieldSet.is_applicable()`.
code_constraints = InterpreterConstraints.create_from_targets(
(tgt for tgt in all_tgts if not tgt.get(SkipBlackField).value),
python_setup)
if code_constraints.requires_python38_or_newer(
python_setup.interpreter_universe):
constraints = code_constraints
return PythonLockfileRequest.from_tool(black, constraints)
def create_request(
name: str,
lockfile_dest: str | None = None) -> PythonLockfileRequest:
return PythonLockfileRequest(
FrozenOrderedSet(),
InterpreterConstraints(),
resolve_name=name,
lockfile_dest=lockfile_dest or f"{name}.txt",
)
async def setup_black_lockfile(
_: BlackLockfileSentinel, black: Black,
python_setup: PythonSetup) -> PythonLockfileRequest:
if not black.uses_lockfile:
return PythonLockfileRequest.from_tool(black)
constraints = black.interpreter_constraints
if black.options.is_default("interpreter_constraints"):
all_build_targets = await Get(UnexpandedTargets,
AddressSpecs([DescendantAddresses("")]))
code_constraints = InterpreterConstraints.create_from_targets(
(tgt for tgt in all_build_targets
if not tgt.get(SkipBlackField).value), python_setup)
if code_constraints.requires_python38_or_newer(
python_setup.interpreter_universe):
constraints = code_constraints
return PythonLockfileRequest.from_tool(black, constraints)
async def setup_bandit_lockfile(
_: BanditLockfileSentinel, bandit: Bandit,
python_setup: PythonSetup) -> PythonLockfileRequest:
if not bandit.uses_lockfile:
return PythonLockfileRequest.from_tool(bandit)
# While Bandit will run in partitions, we need a single lockfile that works with every
# partition.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
unique_constraints = {
InterpreterConstraints.create_from_targets([tgt], python_setup)
for tgt in all_tgts if BanditFieldSet.is_applicable(tgt)
}
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return PythonLockfileRequest.from_tool(
bandit, constraints
or InterpreterConstraints(python_setup.interpreter_constraints))
async def setup_setuptools_lockfile(
_: SetuptoolsLockfileSentinel, setuptools: Setuptools,
python_setup: PythonSetup) -> PythonLockfileRequest:
if not setuptools.uses_lockfile:
return PythonLockfileRequest.from_tool(setuptools)
all_tgts = await Get(AllTargets, AllTargetsRequest())
transitive_targets_per_python_dist = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_tgts if PythonDistributionFieldSet.is_applicable(tgt))
unique_constraints = {
InterpreterConstraints.create_from_targets(transitive_targets.closure,
python_setup)
or InterpreterConstraints(python_setup.interpreter_constraints)
for transitive_targets in transitive_targets_per_python_dist
}
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return PythonLockfileRequest.from_tool(
setuptools, constraints
or InterpreterConstraints(python_setup.interpreter_constraints))
async def setup_flake8_lockfile(
_: Flake8LockfileSentinel, flake8: Flake8, python_setup: PythonSetup
) -> PythonLockfileRequest:
if not flake8.uses_lockfile:
return PythonLockfileRequest.from_tool(flake8)
# While Flake8 will run in partitions, we need a single lockfile that works with every
# partition.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_build_targets = await Get(UnexpandedTargets, AddressSpecs([DescendantAddresses("")]))
unique_constraints = {
InterpreterConstraints.create_from_targets([tgt], python_setup)
for tgt in all_build_targets
if Flake8FieldSet.is_applicable(tgt)
}
constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints))
return PythonLockfileRequest.from_tool(
flake8, constraints or InterpreterConstraints(python_setup.interpreter_constraints)
)
async def generate_user_lockfile_goal(
addresses: Addresses,
python_setup: PythonSetup,
workspace: Workspace,
) -> GenerateUserLockfileGoal:
if python_setup.lockfile is None:
logger.warning(
"You ran `./pants generate-user-lockfile`, but `[python].experimental_lockfile` "
"is not set. Please set this option to the path where you'd like the lockfile for "
"your code's dependencies to live."
)
return GenerateUserLockfileGoal(exit_code=1)
transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(addresses))
reqs = PexRequirements.create_from_requirement_fields(
tgt[PythonRequirementsField]
# NB: By looking at the dependencies, rather than the closure, we only generate for
# requirements that are actually used in the project.
for tgt in transitive_targets.dependencies
if tgt.has_field(PythonRequirementsField)
)
if not reqs:
logger.warning(
"No third-party requirements found for the transitive closure, so a lockfile will not "
"be generated."
)
return GenerateUserLockfileGoal(exit_code=0)
result = await Get(
PythonLockfile,
PythonLockfileRequest(
reqs.req_strings,
# TODO(#12314): Use interpreter constraints from the transitive closure.
InterpreterConstraints(python_setup.interpreter_constraints),
resolve_name="not yet implemented",
lockfile_dest=python_setup.lockfile,
_description=(
f"Generate lockfile for {pluralize(len(reqs.req_strings), 'requirement')}: "
f"{', '.join(reqs.req_strings)}"
),
# TODO(12382): Make this command actually accurate once we figure out the semantics
# for user lockfiles. This is currently misleading.
_regenerate_command="./pants generate-user-lockfile ::",
),
)
workspace.write_digest(result.digest)
logger.info(f"Wrote lockfile to {result.path}")
return GenerateUserLockfileGoal(exit_code=0)
def setup_coverage_lockfile(
_: CoveragePyLockfileSentinel,
coverage: CoverageSubsystem) -> PythonLockfileRequest:
return PythonLockfileRequest.from_tool(coverage)
def setup_yapf_lockfile(_: YapfLockfileSentinel,
yapf: Yapf) -> PythonLockfileRequest:
return PythonLockfileRequest.from_tool(yapf)
def setup_lockfile_request(
_: DocformatterLockfileSentinel,
docformatter: Docformatter) -> PythonLockfileRequest:
return PythonLockfileRequest.from_tool(docformatter)
def setup_mypy_protobuf_lockfile(
_: MypyProtobufLockfileSentinel,
mypy_protobuf: PythonProtobufMypyPlugin) -> PythonLockfileRequest:
return PythonLockfileRequest.from_tool(mypy_protobuf)
def setup_lambdex_lockfile(_: LambdexLockfileSentinel, lambdex: Lambdex) -> PythonLockfileRequest:
return PythonLockfileRequest.from_tool(lambdex)
def setup_lockfile_request(
_: DockerfileParserLockfileSentinel, dockerfile_parser: DockerfileParser
) -> PythonLockfileRequest:
return PythonLockfileRequest.from_tool(dockerfile_parser)
def setup_isort_lockfile(_: IsortLockfileSentinel,
isort: Isort) -> PythonLockfileRequest:
return PythonLockfileRequest.from_tool(isort)
async def setup_autoflake_lockfile(
_: AutoflakeLockfileSentinel,
autoflake: Autoflake) -> PythonLockfileRequest:
return PythonLockfileRequest.from_tool(autoflake)
def setup_pyupgrade_lockfile(_: PyUpgradeLockfileSentinel,
pyupgrade: PyUpgrade) -> PythonLockfileRequest:
return PythonLockfileRequest.from_tool(pyupgrade)
def setup_lockfile_request(
_: TerraformHcl2ParserLockfileSentinel, hcl2_parser: TerraformHcl2Parser
) -> PythonLockfileRequest:
return PythonLockfileRequest.from_tool(hcl2_parser)