def test_build_local_dists(rule_runner: RuleRunner) -> None:
foo = PurePath("foo")
rule_runner.write_files({
foo / "BUILD":
dedent("""
python_sources()
python_distribution(
name = "dist",
dependencies = [":foo"],
provides = python_artifact(name="foo", version="9.8.7"),
sdist = False,
generate_setup = False,
)
"""),
foo / "bar.py":
"BAR = 42",
foo / "setup.py":
dedent("""
from setuptools import setup
setup(name="foo", version="9.8.7", packages=["foo"], package_dir={"foo": "."},)
"""),
})
rule_runner.set_options([], env_inherit={"PATH"})
sources_digest = rule_runner.request(
Digest,
[
CreateDigest([
FileContent("srcroot/foo/bar.py", b""),
FileContent("srcroot/foo/qux.py", b"")
])
],
)
sources_snapshot = rule_runner.request(Snapshot, [sources_digest])
sources = PythonSourceFiles(SourceFiles(sources_snapshot, tuple()),
("srcroot", ))
request = LocalDistsPexRequest([Address("foo", target_name="dist")],
internal_only=True,
sources=sources)
result = rule_runner.request(LocalDistsPex, [request])
assert result.pex is not None
contents = rule_runner.request(DigestContents, [result.pex.digest])
whl_content = None
for content in contents:
if content.path == "local_dists.pex/.deps/foo-9.8.7-py3-none-any.whl":
whl_content = content
assert whl_content
with io.BytesIO(whl_content.content) as fp:
with zipfile.ZipFile(fp, "r") as whl:
assert "foo/bar.py" in whl.namelist()
# Check that srcroot/foo/bar.py was subtracted out, because the dist provides foo/bar.py.
assert result.remaining_sources.source_files.files == (
"srcroot/foo/qux.py", )
async def create_python_repl_request(request: PythonRepl,
pex_env: PexEnvironment,
python_setup: PythonSetup) -> ReplRequest:
validate_compatible_resolve(request.targets, python_setup)
interpreter_constraints, transitive_targets = await MultiGet(
Get(InterpreterConstraints,
InterpreterConstraintsRequest(request.addresses)),
Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)),
)
requirements_request = Get(Pex, RequirementsPexRequest(request.addresses))
local_dists_request = Get(
LocalDistsPex,
LocalDistsPexRequest(
request.addresses,
internal_only=True,
interpreter_constraints=interpreter_constraints,
),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
requirements_pex, local_dists, sources = await MultiGet(
requirements_request, local_dists_request, sources_request)
merged_digest = await Get(
Digest,
MergeDigests((requirements_pex.digest, local_dists.pex.digest,
sources.source_files.snapshot.digest)),
)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(request.in_chroot(
requirements_pex.name),
python=requirements_pex.python)
chrooted_source_roots = [
request.in_chroot(sr) for sr in sources.source_roots
]
extra_env = {
**complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
"PEX_PATH":
request.in_chroot(local_dists.pex.name),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_python_repl_request(repl: PythonRepl,
pex_env: PexEnvironment) -> ReplRequest:
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex) so
# that we can get the interpreter constraints for use in local_dists_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in repl.targets), internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
local_dists_request = Get(
LocalDistsPex,
LocalDistsPexRequest(
Addresses(tgt.address for tgt in repl.targets),
internal_only=True,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(repl.targets, include_files=True))
requirements_pex, local_dists, sources = await MultiGet(
requirements_request, local_dists_request, sources_request)
merged_digest = await Get(
Digest,
MergeDigests((requirements_pex.digest, local_dists.pex.digest,
sources.source_files.snapshot.digest)),
)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(repl.in_chroot(requirements_pex.name),
python=requirements_pex.python)
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
"PEX_PATH":
repl.in_chroot(local_dists.pex.name),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment,
python_setup: PythonSetup,
) -> RunRequest:
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in local_dists_get.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest(
[field_set.address],
output_filename=f"{field_set.address.target_name}.pex",
internal_only=True,
include_source_files=False,
# Note that the file for first-party entry points is not in the PEX itself. In that
# case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
main=entry_point.val or field_set.script.value,
resolve_and_lockfile=field_set.resolve.resolve_and_lockfile(
python_setup),
additional_args=(
*field_set.generate_additional_args(pex_binary_defaults),
# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
# below, it's important for any app that re-executes itself that these environment
# variables are not stripped.
"--no-strip-pex-env",
),
),
)
pex_get = Get(Pex, PexRequest, requirements_pex_request)
sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
pex, sources = await MultiGet(pex_get, sources_get)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[field_set.address],
internal_only=True,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
sources=sources,
),
)
merged_digest = await Get(
Digest,
MergeDigests([
pex.digest,
local_dists.pex.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
"PEX_PATH":
in_chroot(local_dists.pex.name),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
pex_env: PexEnvironment) -> RunRequest:
run_in_sandbox = field_set.run_in_sandbox.value
entry_point, transitive_targets = await MultiGet(
Get(
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest(field_set.entry_point),
),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
addresses = [field_set.address]
interpreter_constraints = await Get(
InterpreterConstraints, InterpreterConstraintsRequest(addresses))
pex_filename = (field_set.address.generated_name.replace(".", "_")
if field_set.address.generated_name else
field_set.address.target_name)
pex_get = Get(
Pex,
PexFromTargetsRequest(
[field_set.address],
output_filename=f"{pex_filename}.pex",
internal_only=True,
include_source_files=False,
# Note that the file for first-party entry points is not in the PEX itself. In that
# case, it's loaded by setting `PEX_EXTRA_SYS_PATH`.
main=entry_point.val or field_set.script.value,
additional_args=(
*field_set.generate_additional_args(pex_binary_defaults),
# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
# below, it's important for any app that re-executes itself that these environment
# variables are not stripped.
"--no-strip-pex-env",
),
),
)
sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
pex, sources = await MultiGet(pex_get, sources_get)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[field_set.address],
internal_only=True,
interpreter_constraints=interpreter_constraints,
sources=sources,
),
)
input_digests = [
pex.digest,
local_dists.pex.digest,
# Note regarding inline mode: You might think that the sources don't need to be copied
# into the chroot when using inline sources. But they do, because some of them might be
# codegenned, and those won't exist in the inline source tree. Rather than incurring the
# complexity of figuring out here which sources were codegenned, we copy everything.
# The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline
# sources will take precedence and their copies in the chroot will be ignored.
local_dists.remaining_sources.source_files.snapshot.digest,
]
merged_digest = await Get(Digest, MergeDigests(input_digests))
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
# The order here is important: we want the in-repo sources to take precedence over their
# copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
source_roots = [
*([] if run_in_sandbox else sources.source_roots),
*chrooted_source_roots,
]
extra_env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
"PEX_PATH":
in_chroot(local_dists.pex.name),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_ipython_repl_request(repl: IPythonRepl, ipython: IPython,
pex_env: PexEnvironment) -> ReplRequest:
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex) so
# that we can get the interpreter constraints for use in ipython_request/local_dists_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(tgt.address for tgt in repl.targets), internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(repl.targets, include_files=True))
ipython_request = Get(
Pex,
PexRequest(
output_filename="ipython.pex",
main=ipython.main,
requirements=ipython.pex_requirements(),
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
internal_only=True,
),
)
requirements_pex, sources, ipython_pex = await MultiGet(
requirements_request, sources_request, ipython_request)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[tgt.address for tgt in repl.targets],
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
sources=sources,
),
)
merged_digest = await Get(
Digest,
MergeDigests((
requirements_pex.digest,
local_dists.pex.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
ipython_pex.digest,
)),
)
complete_pex_env = pex_env.in_workspace()
args = list(
complete_pex_env.create_argv(repl.in_chroot(ipython_pex.name),
python=ipython_pex.python))
if ipython.options.ignore_cwd:
args.append("--ignore-cwd")
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=ipython_pex.python is not None),
"PEX_PATH":
os.pathsep.join([
repl.in_chroot(requirements_pex_request.output_filename),
repl.in_chroot(local_dists.pex.name),
]),
"PEX_EXTRA_SYS_PATH":
os.pathsep.join(chrooted_source_roots),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pytest_for_target(
request: TestSetupRequest,
pytest: PyTest,
test_subsystem: TestSubsystem,
python_setup: PythonSetup,
coverage_config: CoverageConfig,
coverage_subsystem: CoverageSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
complete_env: CompleteEnvironment,
) -> TestSetup:
transitive_targets, plugin_setups = await MultiGet(
Get(TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address])),
Get(AllPytestPluginSetups,
AllPytestPluginSetupsRequest(request.field_set.address)),
)
all_targets = transitive_targets.closure
interpreter_constraints = InterpreterConstraints.create_from_targets(
all_targets, python_setup)
requirements_pex_get = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
[request.field_set.address],
internal_only=True,
resolve_and_lockfile=request.field_set.resolve.
resolve_and_lockfile(python_setup),
),
)
pytest_pex_get = Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=pytest.pex_requirements(),
interpreter_constraints=interpreter_constraints,
internal_only=True,
),
)
# Ensure that the empty extra output dir exists.
extra_output_directory_digest_get = Get(
Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)]))
prepared_sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets, include_files=True))
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
field_set_source_files_get = Get(
SourceFiles, SourceFilesRequest([request.field_set.sources]))
(
pytest_pex,
requirements_pex,
prepared_sources,
field_set_source_files,
extra_output_directory_digest,
) = await MultiGet(
pytest_pex_get,
requirements_pex_get,
prepared_sources_get,
field_set_source_files_get,
extra_output_directory_digest_get,
)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[request.field_set.address],
interpreter_constraints=interpreter_constraints,
sources=prepared_sources,
),
)
pytest_runner_pex_get = Get(
VenvPex,
PexRequest(
output_filename="pytest_runner.pex",
interpreter_constraints=interpreter_constraints,
main=pytest.main,
internal_only=True,
pex_path=[pytest_pex, requirements_pex, local_dists.pex],
),
)
config_files_get = Get(
ConfigFiles,
ConfigFilesRequest,
pytest.config_request(field_set_source_files.snapshot.dirs),
)
pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get,
config_files_get)
input_digest = await Get(
Digest,
MergeDigests((
coverage_config.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
config_files.snapshot.digest,
extra_output_directory_digest,
*(plugin_setup.digest for plugin_setup in plugin_setups),
)),
)
add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"]
output_files = []
results_file_name = None
if pytest.options.junit_xml_dir and not request.is_debug:
results_file_name = f"{request.field_set.address.path_safe_spec}.xml"
add_opts.extend((f"--junitxml={results_file_name}", "-o",
f"junit_family={pytest.options.junit_family}"))
output_files.append(results_file_name)
coverage_args = []
if test_subsystem.use_coverage and not request.is_debug:
pytest.validate_pytest_cov_included()
output_files.append(".coverage")
if coverage_subsystem.filter:
cov_args = [f"--cov={morf}" for morf in coverage_subsystem.filter]
else:
# N.B.: Passing `--cov=` or `--cov=.` to communicate "record coverage for all sources"
# fails in certain contexts as detailed in:
# https://github.com/pantsbuild/pants/issues/12390
# Instead we focus coverage on just the directories containing python source files
# materialized to the Process chroot.
cov_args = [
f"--cov={source_root}"
for source_root in prepared_sources.source_roots
]
coverage_args = [
"--cov-report=", # Turn off output.
f"--cov-config={coverage_config.path}",
*cov_args,
]
extra_env = {
"PYTEST_ADDOPTS": " ".join(add_opts),
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
**test_extra_env.env,
# NOTE: `complete_env` intentionally after `test_extra_env` to allow overriding within
# `python_tests`
**complete_env.get_subset(request.field_set.extra_env_vars.value or ()),
}
# Cache test runs only if they are successful, or not at all if `--test-force`.
cache_scope = (ProcessCacheScope.PER_SESSION
if test_subsystem.force else ProcessCacheScope.SUCCESSFUL)
process = await Get(
Process,
VenvPexProcess(
pytest_runner_pex,
argv=(*pytest.options.args, *coverage_args,
*field_set_source_files.files),
extra_env=extra_env,
input_digest=input_digest,
output_directories=(_EXTRA_OUTPUT_DIR, ),
output_files=output_files,
timeout_seconds=request.field_set.timeout.
calculate_from_global_options(pytest),
execution_slot_variable=pytest.options.execution_slot_var,
description=f"Run Pytest for {request.field_set.address}",
level=LogLevel.DEBUG,
cache_scope=cache_scope,
),
)
return TestSetup(process, results_file_name=results_file_name)
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest:
if request.direct_deps_only:
targets = await Get(Targets, Addresses(request.addresses))
direct_deps = await MultiGet(
Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets
)
all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets))
else:
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest(request.addresses)
)
all_targets = transitive_targets.closure
if request.hardcoded_interpreter_constraints:
interpreter_constraints = request.hardcoded_interpreter_constraints
else:
calculated_constraints = InterpreterConstraints.create_from_targets(
all_targets, python_setup
)
# If there are no targets, we fall back to the global constraints. This is relevant,
# for example, when running `./pants repl` with no specs.
interpreter_constraints = calculated_constraints or InterpreterConstraints(
python_setup.interpreter_constraints
)
sources_digests = []
if request.additional_sources:
sources_digests.append(request.additional_sources)
if request.include_source_files:
sources = await Get(PythonSourceFiles, PythonSourceFilesRequest(all_targets))
else:
sources = PythonSourceFiles.empty()
additional_inputs_digests = []
if request.additional_inputs:
additional_inputs_digests.append(request.additional_inputs)
additional_args = request.additional_args
if request.include_local_dists:
# Note that LocalDistsPexRequest has no `direct_deps_only` mode, so we will build all
# local dists in the transitive closure even if the request was for direct_deps_only.
# Since we currently use `direct_deps_only` in one case (building a requirements pex
# when running pylint) and in that case include_local_dists=False, this seems harmless.
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
request.addresses,
internal_only=request.internal_only,
interpreter_constraints=interpreter_constraints,
sources=sources,
),
)
remaining_sources = local_dists.remaining_sources
additional_inputs_digests.append(local_dists.pex.digest)
additional_args += ("--requirements-pex", local_dists.pex.name)
else:
remaining_sources = sources
remaining_sources_stripped = await Get(
StrippedPythonSourceFiles, PythonSourceFiles, remaining_sources
)
sources_digests.append(remaining_sources_stripped.stripped_source_files.snapshot.digest)
merged_sources_digest, additional_inputs = await MultiGet(
Get(Digest, MergeDigests(sources_digests)),
Get(Digest, MergeDigests(additional_inputs_digests)),
)
requirements = PexRequirements.create_from_requirement_fields(
(
tgt[PythonRequirementsField]
for tgt in all_targets
if tgt.has_field(PythonRequirementsField)
),
additional_requirements=request.additional_requirements,
apply_constraints=True,
)
description = request.description
if requirements:
repository_pex: Pex | None = None
if python_setup.requirement_constraints:
maybe_constraints_repository_pex = await Get(
_ConstraintsRepositoryPex,
_ConstraintsRepositoryPexRequest(
requirements,
request.platforms,
interpreter_constraints,
request.internal_only,
request.additional_lockfile_args,
),
)
if maybe_constraints_repository_pex.maybe_pex:
repository_pex = maybe_constraints_repository_pex.maybe_pex
elif (
python_setup.resolve_all_constraints
and python_setup.resolve_all_constraints_was_set_explicitly()
):
raise ValueError(
"`[python].resolve_all_constraints` is enabled, so "
"`[python].requirement_constraints` must also be set."
)
elif request.resolve_and_lockfile:
resolve, lockfile = request.resolve_and_lockfile
repository_pex = await Get(
Pex,
PexRequest(
description=f"Installing {lockfile} for the resolve `{resolve}`",
output_filename=f"{path_safe(resolve)}_lockfile.pex",
internal_only=request.internal_only,
requirements=Lockfile(
file_path=lockfile,
file_path_description_of_origin=(
f"the resolve `{resolve}` (from "
"`[python].experimental_resolves_to_lockfiles`)"
),
# TODO(#12314): Hook up lockfile staleness check.
lockfile_hex_digest=None,
req_strings=None,
),
interpreter_constraints=interpreter_constraints,
platforms=request.platforms,
additional_args=request.additional_lockfile_args,
),
)
elif python_setup.lockfile:
repository_pex = await Get(
Pex,
PexRequest(
description=f"Installing {python_setup.lockfile}",
output_filename="lockfile.pex",
internal_only=request.internal_only,
requirements=Lockfile(
file_path=python_setup.lockfile,
file_path_description_of_origin=(
"the option `[python].experimental_lockfile`"
),
# TODO(#12314): Hook up lockfile staleness check once multiple lockfiles
# are supported.
lockfile_hex_digest=None,
req_strings=None,
),
interpreter_constraints=interpreter_constraints,
platforms=request.platforms,
additional_args=request.additional_lockfile_args,
),
)
requirements = dataclasses.replace(requirements, repository_pex=repository_pex)
return PexRequest(
output_filename=request.output_filename,
internal_only=request.internal_only,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=request.platforms,
main=request.main,
sources=merged_sources_digest,
additional_inputs=additional_inputs,
additional_args=additional_args,
description=description,
)
async def create_pex_from_targets(request: PexFromTargetsRequest) -> PexRequest:
interpreter_constraints = await Get(
InterpreterConstraints,
InterpreterConstraintsRequest,
request.to_interpreter_constraints_request(),
)
transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses))
sources_digests = []
if request.additional_sources:
sources_digests.append(request.additional_sources)
if request.include_source_files:
sources = await Get(PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure))
else:
sources = PythonSourceFiles.empty()
additional_inputs_digests = []
if request.additional_inputs:
additional_inputs_digests.append(request.additional_inputs)
additional_args = request.additional_args
if request.include_local_dists:
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
request.addresses,
internal_only=request.internal_only,
interpreter_constraints=interpreter_constraints,
sources=sources,
),
)
remaining_sources = local_dists.remaining_sources
additional_inputs_digests.append(local_dists.pex.digest)
additional_args += ("--requirements-pex", local_dists.pex.name)
else:
remaining_sources = sources
remaining_sources_stripped = await Get(
StrippedPythonSourceFiles, PythonSourceFiles, remaining_sources
)
sources_digests.append(remaining_sources_stripped.stripped_source_files.snapshot.digest)
merged_sources_digest, additional_inputs = await MultiGet(
Get(Digest, MergeDigests(sources_digests)),
Get(Digest, MergeDigests(additional_inputs_digests)),
)
description = request.description
if request.include_requirements:
requirements = await Get(PexRequirements, _PexRequirementsRequest(request.addresses))
else:
requirements = PexRequirements()
if requirements:
repository_pex = await Get(
OptionalPex,
_RepositoryPexRequest(
request.addresses,
requirements=requirements,
hardcoded_interpreter_constraints=request.hardcoded_interpreter_constraints,
platforms=request.platforms,
complete_platforms=request.complete_platforms,
internal_only=request.internal_only,
additional_lockfile_args=request.additional_lockfile_args,
),
)
requirements = dataclasses.replace(requirements, repository_pex=repository_pex.maybe_pex)
return PexRequest(
output_filename=request.output_filename,
internal_only=request.internal_only,
layout=request.layout,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=request.platforms,
complete_platforms=request.complete_platforms,
main=request.main,
sources=merged_sources_digest,
additional_inputs=additional_inputs,
additional_args=additional_args,
description=description,
)
async def create_pex_from_targets(request: PexFromTargetsRequest,
python_setup: PythonSetup) -> PexRequest:
requirements: PexRequirements | EntireLockfile = PexRequirements()
if request.include_requirements:
requirements = await Get(PexRequirements,
_PexRequirementsRequest(request.addresses))
pex_native_subsetting_supported = False
if python_setup.enable_resolves:
# TODO: Once `requirement_constraints` is removed in favor of `enable_resolves`,
# `ChosenPythonResolveRequest` and `_PexRequirementsRequest` should merge and
# do a single transitive walk to replace this method.
chosen_resolve = await Get(
ChosenPythonResolve,
ChosenPythonResolveRequest(request.addresses))
loaded_lockfile = await Get(
LoadedLockfile, LoadedLockfileRequest(chosen_resolve.lockfile))
pex_native_subsetting_supported = loaded_lockfile.is_pex_native
if loaded_lockfile.constraints_strings:
requirements = dataclasses.replace(
requirements,
constraints_strings=loaded_lockfile.constraints_strings)
should_return_entire_lockfile = (
python_setup.run_against_entire_lockfile and request.internal_only)
should_request_repository_pex = (
# The entire lockfile was explicitly requested.
should_return_entire_lockfile
# The legacy `resolve_all_constraints`+`requirement_constraints` options were used.
or (
# TODO: The constraints.txt resolve for `resolve_all_constraints` will be removed as
# part of #12314.
python_setup.resolve_all_constraints
and python_setup.requirement_constraints)
# A non-PEX-native lockfile was used, and so we cannot subset it.
or not pex_native_subsetting_supported)
if should_request_repository_pex:
repository_pex_request = await Get(
OptionalPexRequest,
_RepositoryPexRequest(
request.addresses,
hardcoded_interpreter_constraints=request.
hardcoded_interpreter_constraints,
platforms=request.platforms,
complete_platforms=request.complete_platforms,
internal_only=request.internal_only,
additional_lockfile_args=request.additional_lockfile_args,
),
)
if should_return_entire_lockfile:
if repository_pex_request.maybe_pex_request is None:
raise ValueError(
softwrap(f"""
[python].run_against_entire_lockfile was set, but could not find a
lockfile or constraints file for this target set. See
{doc_url('python-third-party-dependencies')} for details.
"""))
return repository_pex_request.maybe_pex_request
repository_pex = await Get(OptionalPex, OptionalPexRequest,
repository_pex_request)
requirements = dataclasses.replace(
requirements, from_superset=repository_pex.maybe_pex)
elif python_setup.enable_resolves:
# NB: We confirmed above that this is a PEX-native lockfile, so it can be used as a
# superset.
chosen_resolve = await Get(
ChosenPythonResolve,
ChosenPythonResolveRequest(request.addresses))
loaded_lockfile = await Get(
LoadedLockfile, LoadedLockfileRequest(chosen_resolve.lockfile))
requirements = dataclasses.replace(requirements,
from_superset=loaded_lockfile)
interpreter_constraints = await Get(
InterpreterConstraints,
InterpreterConstraintsRequest,
request.to_interpreter_constraints_request(),
)
transitive_targets = await Get(TransitiveTargets,
TransitiveTargetsRequest(request.addresses))
sources_digests = []
if request.additional_sources:
sources_digests.append(request.additional_sources)
if request.include_source_files:
sources = await Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure))
else:
sources = PythonSourceFiles.empty()
additional_inputs_digests = []
if request.additional_inputs:
additional_inputs_digests.append(request.additional_inputs)
additional_args = request.additional_args
if request.include_local_dists:
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
request.addresses,
internal_only=request.internal_only,
interpreter_constraints=interpreter_constraints,
sources=sources,
),
)
remaining_sources = local_dists.remaining_sources
additional_inputs_digests.append(local_dists.pex.digest)
additional_args += ("--requirements-pex", local_dists.pex.name)
else:
remaining_sources = sources
remaining_sources_stripped = await Get(StrippedPythonSourceFiles,
PythonSourceFiles,
remaining_sources)
sources_digests.append(
remaining_sources_stripped.stripped_source_files.snapshot.digest)
merged_sources_digest, additional_inputs = await MultiGet(
Get(Digest, MergeDigests(sources_digests)),
Get(Digest, MergeDigests(additional_inputs_digests)),
)
description = request.description
return PexRequest(
output_filename=request.output_filename,
internal_only=request.internal_only,
layout=request.layout,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=request.platforms,
complete_platforms=request.complete_platforms,
main=request.main,
sources=merged_sources_digest,
additional_inputs=additional_inputs,
additional_args=additional_args,
description=description,
)