async def setup_pytest_lockfile(
_: PytestLockfileSentinel, pytest: PyTest,
python_setup: PythonSetup) -> GeneratePythonLockfile:
if not pytest.uses_lockfile:
return GeneratePythonLockfile.from_tool(
pytest, use_pex=python_setup.generate_lockfiles_with_pex)
# Even though we run each python_tests target in isolation, we need a single lockfile that
# works with them all (and their transitive deps).
#
# This first computes the constraints for each individual `python_test` target
# (which will AND across each target in the closure). Then, it ORs all unique resulting
# interpreter constraints. The net effect is that every possible Python interpreter used will
# be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
transitive_targets_per_test = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_tgts if PythonTestFieldSet.is_applicable(tgt))
unique_constraints = {
InterpreterConstraints.create_from_targets(transitive_targets.closure,
python_setup)
for transitive_targets in transitive_targets_per_test
}
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
pytest,
constraints
or InterpreterConstraints(python_setup.interpreter_constraints),
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_bandit_lockfile(
_: BanditLockfileSentinel, bandit: Bandit, python_setup: PythonSetup
) -> GeneratePythonLockfile:
if not bandit.uses_lockfile:
return GeneratePythonLockfile.from_tool(
bandit, use_pex=python_setup.generate_lockfiles_with_pex
)
# While Bandit will run in partitions, we need a single lockfile that works with every
# partition.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
unique_constraints = {
InterpreterConstraints.create_from_targets([tgt], python_setup)
for tgt in all_tgts
if BanditFieldSet.is_applicable(tgt)
}
constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
bandit,
constraints or InterpreterConstraints(python_setup.interpreter_constraints),
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_mypy_lockfile(
_: MyPyLockfileSentinel,
first_party_plugins: MyPyFirstPartyPlugins,
mypy: MyPy,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
if not mypy.uses_lockfile:
return GeneratePythonLockfile.from_tool(
mypy, use_pex=python_setup.generate_lockfiles_with_pex)
constraints = mypy.interpreter_constraints
if mypy.options.is_default("interpreter_constraints"):
all_tgts = await Get(AllTargets, AllTargetsRequest())
all_transitive_targets = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_tgts if MyPyFieldSet.is_applicable(tgt))
unique_constraints = {
InterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup)
for transitive_targets in all_transitive_targets
}
code_constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
if code_constraints.requires_python38_or_newer(
python_setup.interpreter_universe):
constraints = code_constraints
return GeneratePythonLockfile.from_tool(
mypy,
constraints,
extra_requirements=first_party_plugins.requirement_strings,
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_setuptools_lockfile(
_: SetuptoolsLockfileSentinel, setuptools: Setuptools, python_setup: PythonSetup
) -> GeneratePythonLockfile:
if not setuptools.uses_custom_lockfile:
return GeneratePythonLockfile.from_tool(
setuptools, use_pex=python_setup.generate_lockfiles_with_pex
)
all_tgts = await Get(AllTargets, AllTargetsRequest())
transitive_targets_per_python_dist = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_tgts
if PythonDistributionFieldSet.is_applicable(tgt)
)
unique_constraints = {
InterpreterConstraints.create_from_targets(transitive_targets.closure, python_setup)
or InterpreterConstraints(python_setup.interpreter_constraints)
for transitive_targets in transitive_targets_per_python_dist
}
constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
setuptools,
constraints or InterpreterConstraints(python_setup.interpreter_constraints),
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_ipython_lockfile(
_: IPythonLockfileSentinel, ipython: IPython,
python_setup: PythonSetup) -> GeneratePythonLockfile:
if not ipython.uses_lockfile:
return GeneratePythonLockfile.from_tool(
ipython, use_pex=python_setup.generate_lockfiles_with_pex)
# IPython is often run against the whole repo (`./pants repl ::`), but it is possible to run
# on subsets of the codebase with disjoint interpreter constraints, such as
# `./pants repl py2::` and then `./pants repl py3::`. Still, even with those subsets possible,
# we need a single lockfile that works with all possible Python interpreters in use.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
unique_constraints = {
InterpreterConstraints.create_from_compatibility_fields(
[tgt[InterpreterConstraintsField]], python_setup)
for tgt in all_tgts if tgt.has_field(InterpreterConstraintsField)
}
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
ipython,
constraints
or InterpreterConstraints(python_setup.interpreter_constraints),
use_pex=python_setup.generate_lockfiles_with_pex,
)
def test_multiple_resolves() -> None:
rule_runner = RuleRunner(
rules=[
setup_user_lockfile_requests,
SubsystemRule(PythonSetup),
QueryRule(UserGenerateLockfiles,
[RequestedPythonUserResolveNames]),
],
target_types=[PythonRequirementTarget],
)
rule_runner.write_files({
"BUILD":
dedent("""\
python_requirement(
name='a',
requirements=['a'],
resolve='a',
)
python_requirement(
name='b',
requirements=['b'],
resolve='b',
)
"""),
})
rule_runner.set_options(
[
"--python-resolves={'a': 'a.lock', 'b': 'b.lock'}",
# Override interpreter constraints for 'b', but use default for 'a'.
"--python-resolves-to-interpreter-constraints={'b': ['==3.7.*']}",
"--python-enable-resolves",
"--python-lockfile-generator=pex",
],
env_inherit=PYTHON_BOOTSTRAP_ENV,
)
result = rule_runner.request(UserGenerateLockfiles,
[RequestedPythonUserResolveNames(["a", "b"])])
assert set(result) == {
GeneratePythonLockfile(
requirements=FrozenOrderedSet(["a"]),
interpreter_constraints=InterpreterConstraints(
PythonSetup.default_interpreter_constraints),
resolve_name="a",
lockfile_dest="a.lock",
use_pex=True,
),
GeneratePythonLockfile(
requirements=FrozenOrderedSet(["b"]),
interpreter_constraints=InterpreterConstraints(["==3.7.*"]),
resolve_name="b",
lockfile_dest="b.lock",
use_pex=True,
),
}
async def setup_black_lockfile(
_: BlackLockfileSentinel, black: Black, python_setup: PythonSetup
) -> GeneratePythonLockfile:
if not black.uses_custom_lockfile:
return GeneratePythonLockfile.from_tool(
black, use_pex=python_setup.generate_lockfiles_with_pex
)
constraints = await _black_interpreter_constraints(black, python_setup)
return GeneratePythonLockfile.from_tool(
black, constraints, use_pex=python_setup.generate_lockfiles_with_pex
)
async def setup_pytest_lockfile(
_: PytestLockfileSentinel, pytest: PyTest,
python_setup: PythonSetup) -> GeneratePythonLockfile:
if not pytest.uses_custom_lockfile:
return GeneratePythonLockfile.from_tool(
pytest, use_pex=python_setup.generate_lockfiles_with_pex)
constraints = await _pytest_interpreter_constraints(python_setup)
return GeneratePythonLockfile.from_tool(
pytest,
constraints,
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_pylint_lockfile(
_: PylintLockfileSentinel,
first_party_plugins: PylintFirstPartyPlugins,
pylint: Pylint,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
if not pylint.uses_lockfile:
return GeneratePythonLockfile.from_tool(
pylint, use_pex=python_setup.generate_lockfiles_with_pex)
# While Pylint will run in partitions, we need a single lockfile that works with every
# partition. We must also consider any 3rd-party requirements used by 1st-party plugins.
#
# This first computes the constraints for each individual target, including its direct
# dependencies (which will AND across each target in the closure). Then, it ORs all unique
# resulting interpreter constraints. The net effect is that every possible Python interpreter
# used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
relevant_targets = tuple(tgt for tgt in all_tgts
if PylintFieldSet.is_applicable(tgt))
direct_deps_per_target = await MultiGet(
Get(Targets, DependenciesRequest(tgt.get(Dependencies)))
for tgt in relevant_targets)
unique_constraints = set()
for tgt, direct_deps in zip(relevant_targets, direct_deps_per_target):
constraints_fields = (t[InterpreterConstraintsField]
for t in (tgt, *direct_deps)
if t.has_field(InterpreterConstraintsField))
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
(*constraints_fields,
*first_party_plugins.interpreter_constraints_fields),
python_setup,
))
if not unique_constraints:
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
first_party_plugins.interpreter_constraints_fields,
python_setup,
))
constraints = InterpreterConstraints(
itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
pylint,
constraints
or InterpreterConstraints(python_setup.interpreter_constraints),
extra_requirements=first_party_plugins.requirement_strings,
use_pex=python_setup.generate_lockfiles_with_pex,
)
def setup_mypy_protobuf_lockfile(
_: MypyProtobufLockfileSentinel,
mypy_protobuf: PythonProtobufMypyPlugin,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
mypy_protobuf, use_pex=python_setup.generate_lockfiles_with_pex)
def setup_lockfile_request(
_: DockerfileParserLockfileSentinel,
dockerfile_parser: DockerfileParser,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
dockerfile_parser, use_pex=python_setup.generate_lockfiles_with_pex)
def setup_lockfile_request(
_: TerraformHcl2ParserLockfileSentinel,
hcl2_parser: TerraformHcl2Parser,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
hcl2_parser, use_pex=python_setup.generate_lockfiles_with_pex)
async def setup_mypy_lockfile(
_: MyPyLockfileSentinel,
first_party_plugins: MyPyFirstPartyPlugins,
mypy: MyPy,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
if not mypy.uses_custom_lockfile:
return GeneratePythonLockfile.from_tool(
mypy, use_pex=python_setup.generate_lockfiles_with_pex)
constraints = await _mypy_interpreter_constraints(mypy, python_setup)
return GeneratePythonLockfile.from_tool(
mypy,
constraints,
extra_requirements=first_party_plugins.requirement_strings,
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_pylint_lockfile(
_: PylintLockfileSentinel,
first_party_plugins: PylintFirstPartyPlugins,
pylint: Pylint,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
if not pylint.uses_custom_lockfile:
return GeneratePythonLockfile.from_tool(
pylint, use_pex=python_setup.generate_lockfiles_with_pex
)
constraints = await _pylint_interpreter_constraints(first_party_plugins, python_setup)
return GeneratePythonLockfile.from_tool(
pylint,
constraints,
extra_requirements=first_party_plugins.requirement_strings,
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_flake8_lockfile(
_: Flake8LockfileSentinel,
first_party_plugins: Flake8FirstPartyPlugins,
flake8: Flake8,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
if not flake8.uses_custom_lockfile:
return GeneratePythonLockfile.from_tool(
flake8, use_pex=python_setup.generate_lockfiles_with_pex)
constraints = await _flake8_interpreter_constraints(
first_party_plugins, python_setup)
return GeneratePythonLockfile.from_tool(
flake8,
constraints,
extra_requirements=first_party_plugins.requirement_strings,
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_flake8_lockfile(
_: Flake8LockfileSentinel,
first_party_plugins: Flake8FirstPartyPlugins,
flake8: Flake8,
python_setup: PythonSetup,
) -> GeneratePythonLockfile:
if not flake8.uses_lockfile:
return GeneratePythonLockfile.from_tool(
flake8, use_pex=python_setup.generate_lockfiles_with_pex
)
# While Flake8 will run in partitions, we need a single lockfile that works with every
# partition.
#
# This ORs all unique interpreter constraints. The net effect is that every possible Python
# interpreter used will be covered.
all_tgts = await Get(AllTargets, AllTargetsRequest())
relevant_targets = tuple(tgt for tgt in all_tgts if Flake8FieldSet.is_applicable(tgt))
unique_constraints = set()
for tgt in relevant_targets:
if tgt.has_field(InterpreterConstraintsField):
constraints_field = tgt[InterpreterConstraintsField]
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
(constraints_field, *first_party_plugins.interpreter_constraints_fields),
python_setup,
)
)
if not unique_constraints:
unique_constraints.add(
InterpreterConstraints.create_from_compatibility_fields(
first_party_plugins.interpreter_constraints_fields,
python_setup,
)
)
constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints))
return GeneratePythonLockfile.from_tool(
flake8,
constraints or InterpreterConstraints(python_setup.interpreter_constraints),
extra_requirements=first_party_plugins.requirement_strings,
use_pex=python_setup.generate_lockfiles_with_pex,
)
async def setup_black_lockfile(
_: BlackLockfileSentinel, black: Black,
python_setup: PythonSetup) -> GeneratePythonLockfile:
if not black.uses_lockfile:
return GeneratePythonLockfile.from_tool(
black, use_pex=python_setup.generate_lockfiles_with_pex)
constraints = black.interpreter_constraints
if black.options.is_default("interpreter_constraints"):
all_tgts = await Get(AllTargets, AllTargetsRequest())
# TODO: fix to use `FieldSet.is_applicable()`.
code_constraints = InterpreterConstraints.create_from_targets(
(tgt for tgt in all_tgts if not tgt.get(SkipBlackField).value),
python_setup)
if code_constraints.requires_python38_or_newer(
python_setup.interpreter_universe):
constraints = code_constraints
return GeneratePythonLockfile.from_tool(
black, constraints, use_pex=python_setup.generate_lockfiles_with_pex)
def generate(*, use_pex: bool) -> str:
result = rule_runner.request(
GenerateLockfileResult,
[
GeneratePythonLockfile(
requirements=FrozenOrderedSet(["ansicolors==1.1.8"]),
interpreter_constraints=InterpreterConstraints(),
resolve_name="test",
lockfile_dest="test.lock",
use_pex=use_pex,
)
],
)
digest_contents = rule_runner.request(DigestContents, [result.digest])
assert len(digest_contents) == 1
return digest_contents[0].content.decode()
def setup_lambdex_lockfile(
_: LambdexLockfileSentinel, lambdex: Lambdex,
python_setup: PythonSetup) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
lambdex, use_pex=python_setup.generate_lockfiles_with_pex)
async def setup_autoflake_lockfile(
_: AutoflakeLockfileSentinel, autoflake: Autoflake, python_setup: PythonSetup
) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
autoflake, use_pex=python_setup.generate_lockfiles_with_pex
)
def setup_clangformat_lockfile(
_: ClangFormatLockfileSentinel, clangformat: ClangFormat, python_setup: PythonSetup
) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
clangformat, use_pex=python_setup.generate_lockfiles_with_pex
)
def setup_pyupgrade_lockfile(
_: PyUpgradeLockfileSentinel, pyupgrade: PyUpgrade,
python_setup: PythonSetup) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
pyupgrade, use_pex=python_setup.generate_lockfiles_with_pex)
def setup_yapf_lockfile(
_: YapfLockfileSentinel, yapf: Yapf, python_setup: PythonSetup
) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(yapf, use_pex=python_setup.generate_lockfiles_with_pex)
def setup_twine_lockfile(_: TwineLockfileSentinel, twine: TwineSubsystem,
python_setup: PythonSetup) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
twine, use_pex=python_setup.generate_lockfiles_with_pex)
def setup_coverage_lockfile(
_: CoveragePyLockfileSentinel, coverage: CoverageSubsystem,
python_setup: PythonSetup) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
coverage, use_pex=python_setup.generate_lockfiles_with_pex)
def setup_setuptools_scm_lockfile(
_: SetuptoolsSCMLockfileSentinel, setuptools_scm: SetuptoolsSCM,
python_setup: PythonSetup) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
setuptools_scm, use_pex=python_setup.generate_lockfiles_with_pex)
def setup_isort_lockfile(_: IsortLockfileSentinel, isort: Isort,
python_setup: PythonSetup) -> GeneratePythonLockfile:
return GeneratePythonLockfile.from_tool(
isort, use_pex=python_setup.generate_lockfiles_with_pex)