async def create_python_repl_request(request: PythonRepl,
pex_env: PexEnvironment,
python_setup: PythonSetup) -> ReplRequest:
validate_compatible_resolve(request.targets, python_setup)
interpreter_constraints, transitive_targets = await MultiGet(
Get(InterpreterConstraints,
InterpreterConstraintsRequest(request.addresses)),
Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)),
)
requirements_request = Get(Pex, RequirementsPexRequest(request.addresses))
local_dists_request = Get(
LocalDistsPex,
LocalDistsPexRequest(
request.addresses,
internal_only=True,
interpreter_constraints=interpreter_constraints,
),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
requirements_pex, local_dists, sources = await MultiGet(
requirements_request, local_dists_request, sources_request)
merged_digest = await Get(
Digest,
MergeDigests((requirements_pex.digest, local_dists.pex.digest,
sources.source_files.snapshot.digest)),
)
complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(request.in_chroot(
requirements_pex.name),
python=requirements_pex.python)
chrooted_source_roots = [
request.in_chroot(sr) for sr in sources.source_roots
]
extra_env = {
**complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
"PEX_PATH":
request.in_chroot(local_dists.pex.name),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def get_mypyc_build_environment(
request: MyPycDistBuildEnvironmentRequest,
first_party_plugins: MyPyFirstPartyPlugins,
mypy_config_file: MyPyConfigFile,
mypy: MyPy,
) -> DistBuildEnvironment:
mypy_pex_get = Get(
Pex,
PexRequest,
mypy.to_pex_request(
interpreter_constraints=request.interpreter_constraints,
extra_requirements=first_party_plugins.requirement_strings,
),
)
requirements_pex_get = Get(
Pex,
RequirementsPexRequest(
addresses=request.target_addresses,
hardcoded_interpreter_constraints=request.interpreter_constraints,
),
)
extra_type_stubs_pex_get = Get(
Pex,
PexRequest(
output_filename="extra_type_stubs.pex",
internal_only=True,
requirements=PexRequirements(mypy.extra_type_stubs),
interpreter_constraints=request.interpreter_constraints,
),
)
(mypy_pex, requirements_pex,
extra_type_stubs_pex) = await MultiGet(mypy_pex_get, requirements_pex_get,
extra_type_stubs_pex_get)
return DistBuildEnvironment(
extra_build_time_requirements=(mypy_pex, requirements_pex,
extra_type_stubs_pex),
extra_build_time_inputs=mypy_config_file.digest,
)
async def pylint_lint_partition(
partition: PylintPartition, pylint: Pylint,
first_party_plugins: PylintFirstPartyPlugins) -> LintResult:
requirements_pex_get = Get(
Pex,
RequirementsPexRequest(
(t.address for t in partition.root_targets),
# NB: These constraints must be identical to the other PEXes. Otherwise, we risk using
# a different version for the requirements than the other two PEXes, which can result
# in a PEX runtime error about missing dependencies.
hardcoded_interpreter_constraints=partition.
interpreter_constraints,
internal_only=True,
),
)
pylint_pex_get = Get(
Pex,
PexRequest,
pylint.to_pex_request(
interpreter_constraints=partition.interpreter_constraints,
extra_requirements=first_party_plugins.requirement_strings,
),
)
prepare_python_sources_get = Get(
PythonSourceFiles, PythonSourceFilesRequest(partition.closure))
field_set_sources_get = Get(
SourceFiles,
SourceFilesRequest(t[PythonSourceField]
for t in partition.root_targets))
# Ensure that the empty report dir exists.
report_directory_digest_get = Get(Digest,
CreateDigest([Directory(REPORT_DIR)]))
(
pylint_pex,
requirements_pex,
prepared_python_sources,
field_set_sources,
report_directory,
) = await MultiGet(
pylint_pex_get,
requirements_pex_get,
prepare_python_sources_get,
field_set_sources_get,
report_directory_digest_get,
)
pylint_runner_pex, config_files = await MultiGet(
Get(
VenvPex,
VenvPexRequest(
PexRequest(
output_filename="pylint_runner.pex",
interpreter_constraints=partition.interpreter_constraints,
main=pylint.main,
internal_only=True,
pex_path=[pylint_pex, requirements_pex],
),
# TODO(John Sirois): Remove this (change to the default of symlinks) when we can
# upgrade to a version of Pylint with https://github.com/PyCQA/pylint/issues/1470
# resolved.
site_packages_copies=True,
),
),
Get(ConfigFiles, ConfigFilesRequest,
pylint.config_request(field_set_sources.snapshot.dirs)),
)
pythonpath = list(prepared_python_sources.source_roots)
if first_party_plugins:
pythonpath.append(first_party_plugins.PREFIX)
input_digest = await Get(
Digest,
MergeDigests((
config_files.snapshot.digest,
first_party_plugins.sources_digest,
prepared_python_sources.source_files.snapshot.digest,
report_directory,
)),
)
result = await Get(
FallibleProcessResult,
VenvPexProcess(
pylint_runner_pex,
argv=generate_argv(field_set_sources, pylint),
input_digest=input_digest,
output_directories=(REPORT_DIR, ),
extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},
concurrency_available=len(partition.root_targets),
description=
f"Run Pylint on {pluralize(len(partition.root_targets), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR))
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def collect_fixture_configs(
_request: CollectFixtureConfigsRequest,
pytest: PyTest,
python_setup: PythonSetup,
test_extra_env: TestExtraEnv,
targets: Targets,
) -> CollectedJVMLockfileFixtureConfigs:
addresses = [tgt.address for tgt in targets]
transitive_targets = await Get(TransitiveTargets,
TransitiveTargetsRequest(addresses))
all_targets = transitive_targets.closure
interpreter_constraints = InterpreterConstraints.create_from_targets(
all_targets, python_setup)
pytest_pex, requirements_pex, prepared_sources, root_sources = await MultiGet(
Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=pytest.pex_requirements(),
interpreter_constraints=interpreter_constraints,
internal_only=True,
),
),
Get(Pex, RequirementsPexRequest(addresses)),
Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets,
include_files=True,
include_resources=True),
),
Get(
PythonSourceFiles,
PythonSourceFilesRequest(targets),
),
)
script_content = FileContent(path="collect-fixtures.py",
content=COLLECTION_SCRIPT.encode(),
is_executable=True)
script_digest = await Get(Digest, CreateDigest([script_content]))
pytest_runner_pex_get = Get(
VenvPex,
PexRequest(
output_filename="pytest_runner.pex",
interpreter_constraints=interpreter_constraints,
main=EntryPoint(PurePath(script_content.path).stem),
sources=script_digest,
internal_only=True,
pex_path=[
pytest_pex,
requirements_pex,
],
),
)
config_file_dirs = list(
group_by_dir(prepared_sources.source_files.files).keys())
config_files_get = Get(
ConfigFiles,
ConfigFilesRequest,
pytest.config_request(config_file_dirs),
)
pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get,
config_files_get)
pytest_config_digest = config_files.snapshot.digest
input_digest = await Get(
Digest,
MergeDigests((
prepared_sources.source_files.snapshot.digest,
pytest_config_digest,
)),
)
extra_env = {
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
**test_extra_env.env,
}
process = await Get(
Process,
VenvPexProcess(
pytest_runner_pex,
argv=[
name for name in root_sources.source_files.files
if name.endswith(".py")
],
extra_env=extra_env,
input_digest=input_digest,
output_files=("tests.json", ),
description="Collect test lockfile requirements from all tests.",
level=LogLevel.DEBUG,
cache_scope=ProcessCacheScope.PER_SESSION,
),
)
result = await Get(ProcessResult, Process, process)
digest_contents = await Get(DigestContents, Digest, result.output_digest)
assert len(digest_contents) == 1
assert digest_contents[0].path == "tests.json"
raw_config_data = json.loads(digest_contents[0].content)
configs = []
for item in raw_config_data:
config = JVMLockfileFixtureConfig(
definition=JVMLockfileFixtureDefinition.from_kwargs(
item["kwargs"]),
test_file_path=item["test_file_path"],
)
configs.append(config)
return CollectedJVMLockfileFixtureConfigs(configs)
async def mypy_typecheck_partition(
partition: MyPyPartition,
config_file: MyPyConfigFile,
first_party_plugins: MyPyFirstPartyPlugins,
mypy: MyPy,
python_setup: PythonSetup,
) -> CheckResult:
# MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6,
# and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with
# Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a
# loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if
# 3.8+ is not installed on the machine.
tool_interpreter_constraints = (partition.interpreter_constraints if (
mypy.options.is_default("interpreter_constraints")
and partition.interpreter_constraints.requires_python38_or_newer(
python_setup.interpreter_universe)) else
mypy.interpreter_constraints)
closure_sources_get = Get(PythonSourceFiles,
PythonSourceFilesRequest(partition.closure))
roots_sources_get = Get(
SourceFiles,
SourceFilesRequest(
tgt.get(PythonSourceField) for tgt in partition.root_targets),
)
# See `requirements_venv_pex` for how this will get wrapped in a `VenvPex`.
requirements_pex_get = Get(
Pex,
RequirementsPexRequest(
(tgt.address for tgt in partition.root_targets),
hardcoded_interpreter_constraints=partition.
interpreter_constraints,
internal_only=True,
),
)
extra_type_stubs_pex_get = Get(
Pex,
PexRequest(
output_filename="extra_type_stubs.pex",
internal_only=True,
requirements=PexRequirements(mypy.extra_type_stubs),
interpreter_constraints=partition.interpreter_constraints,
),
)
mypy_pex_get = Get(
VenvPex,
PexRequest,
mypy.to_pex_request(
interpreter_constraints=tool_interpreter_constraints,
extra_requirements=first_party_plugins.requirement_strings,
),
)
(
closure_sources,
roots_sources,
mypy_pex,
extra_type_stubs_pex,
requirements_pex,
) = await MultiGet(
closure_sources_get,
roots_sources_get,
mypy_pex_get,
extra_type_stubs_pex_get,
requirements_pex_get,
)
python_files = determine_python_files(roots_sources.snapshot.files)
file_list_path = "__files.txt"
file_list_digest_request = Get(
Digest,
CreateDigest(
[FileContent(file_list_path, "\n".join(python_files).encode())]),
)
# This creates a venv with all the 3rd-party requirements used by the code. We tell MyPy to
# use this venv by setting `--python-executable`. Note that this Python interpreter is
# different than what we run MyPy with.
#
# We could have directly asked the `PexFromTargetsRequest` to return a `VenvPex`, rather than
# `Pex`, but that would mean missing out on sharing a cache with other goals like `test` and
# `run`.
requirements_venv_pex_request = Get(
VenvPex,
PexRequest(
output_filename="requirements_venv.pex",
internal_only=True,
pex_path=[requirements_pex, extra_type_stubs_pex],
interpreter_constraints=partition.interpreter_constraints,
),
)
requirements_venv_pex, file_list_digest = await MultiGet(
requirements_venv_pex_request, file_list_digest_request)
merged_input_files = await Get(
Digest,
MergeDigests([
file_list_digest,
first_party_plugins.sources_digest,
closure_sources.source_files.snapshot.digest,
requirements_venv_pex.digest,
config_file.digest,
]),
)
all_used_source_roots = sorted(
set(
itertools.chain(first_party_plugins.source_roots,
closure_sources.source_roots)))
env = {
"PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots),
"MYPYPATH": ":".join(all_used_source_roots),
}
result = await Get(
FallibleProcessResult,
VenvPexProcess(
mypy_pex,
argv=generate_argv(
mypy,
venv_python=requirements_venv_pex.python.argv0,
file_list_path=file_list_path,
python_version=config_file.python_version_to_autoset(
partition.interpreter_constraints,
python_setup.interpreter_universe),
),
input_digest=merged_input_files,
extra_env=env,
output_directories=(REPORT_DIR, ),
description=f"Run MyPy on {pluralize(len(python_files), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR))
return CheckResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def setup_pytest_for_target(
request: TestSetupRequest,
pytest: PyTest,
test_subsystem: TestSubsystem,
python_setup: PythonSetup,
coverage_config: CoverageConfig,
coverage_subsystem: CoverageSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
) -> TestSetup:
transitive_targets, plugin_setups = await MultiGet(
Get(TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address])),
Get(AllPytestPluginSetups,
AllPytestPluginSetupsRequest(request.field_set.address)),
)
all_targets = transitive_targets.closure
interpreter_constraints = InterpreterConstraints.create_from_targets(
all_targets, python_setup)
requirements_pex_get = Get(
Pex,
RequirementsPexRequest([request.field_set.address],
internal_only=True))
pytest_pex_get = Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=pytest.pex_requirements(),
interpreter_constraints=interpreter_constraints,
internal_only=True,
),
)
# Ensure that the empty extra output dir exists.
extra_output_directory_digest_get = Get(
Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)]))
prepared_sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets, include_files=True))
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
field_set_source_files_get = Get(
SourceFiles, SourceFilesRequest([request.field_set.source]))
field_set_extra_env_get = Get(
Environment,
EnvironmentRequest(request.field_set.extra_env_vars.value or ()))
(
pytest_pex,
requirements_pex,
prepared_sources,
field_set_source_files,
field_set_extra_env,
extra_output_directory_digest,
) = await MultiGet(
pytest_pex_get,
requirements_pex_get,
prepared_sources_get,
field_set_source_files_get,
field_set_extra_env_get,
extra_output_directory_digest_get,
)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[request.field_set.address],
internal_only=True,
interpreter_constraints=interpreter_constraints,
sources=prepared_sources,
),
)
pytest_runner_pex_get = Get(
VenvPex,
PexRequest(
output_filename="pytest_runner.pex",
interpreter_constraints=interpreter_constraints,
main=pytest.main,
internal_only=True,
pex_path=[pytest_pex, requirements_pex, local_dists.pex],
),
)
config_files_get = Get(
ConfigFiles,
ConfigFilesRequest,
pytest.config_request(field_set_source_files.snapshot.dirs),
)
pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get,
config_files_get)
# The coverage and pytest config may live in the same config file (e.g., setup.cfg, tox.ini
# or pyproject.toml), and wee may have rewritten those files to augment the coverage config,
# in which case we must ensure that the original and rewritten files don't collide.
pytest_config_digest = config_files.snapshot.digest
if coverage_config.path in config_files.snapshot.files:
subset_paths = list(config_files.snapshot.files)
# Remove the original file, and rely on the rewritten file, which contains all the
# pytest-related config unchanged.
subset_paths.remove(coverage_config.path)
pytest_config_digest = await Get(
Digest, DigestSubset(pytest_config_digest,
PathGlobs(subset_paths)))
input_digest = await Get(
Digest,
MergeDigests((
coverage_config.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
pytest_config_digest,
extra_output_directory_digest,
*(plugin_setup.digest for plugin_setup in plugin_setups),
)),
)
add_opts = [f"--color={'yes' if global_options.colors else 'no'}"]
output_files = []
results_file_name = None
if not request.is_debug:
results_file_name = f"{request.field_set.address.path_safe_spec}.xml"
add_opts.extend((f"--junitxml={results_file_name}", "-o",
f"junit_family={pytest.junit_family}"))
output_files.append(results_file_name)
coverage_args = []
if test_subsystem.use_coverage and not request.is_debug:
pytest.validate_pytest_cov_included()
output_files.append(".coverage")
if coverage_subsystem.filter:
cov_args = [f"--cov={morf}" for morf in coverage_subsystem.filter]
else:
# N.B.: Passing `--cov=` or `--cov=.` to communicate "record coverage for all sources"
# fails in certain contexts as detailed in:
# https://github.com/pantsbuild/pants/issues/12390
# Instead we focus coverage on just the directories containing python source files
# materialized to the Process chroot.
cov_args = [
f"--cov={source_root}"
for source_root in prepared_sources.source_roots
]
coverage_args = [
"--cov-report=", # Turn off output.
f"--cov-config={coverage_config.path}",
*cov_args,
]
extra_env = {
"PYTEST_ADDOPTS": " ".join(add_opts),
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
**test_extra_env.env,
# NOTE: field_set_extra_env intentionally after `test_extra_env` to allow overriding within
# `python_tests`.
**field_set_extra_env,
}
# Cache test runs only if they are successful, or not at all if `--test-force`.
cache_scope = (ProcessCacheScope.PER_SESSION
if test_subsystem.force else ProcessCacheScope.SUCCESSFUL)
process = await Get(
Process,
VenvPexProcess(
pytest_runner_pex,
argv=(*pytest.args, *coverage_args, *field_set_source_files.files),
extra_env=extra_env,
input_digest=input_digest,
output_directories=(_EXTRA_OUTPUT_DIR, ),
output_files=output_files,
timeout_seconds=request.field_set.timeout.
calculate_from_global_options(pytest),
execution_slot_variable=pytest.execution_slot_var,
description=f"Run Pytest for {request.field_set.address}",
level=LogLevel.DEBUG,
cache_scope=cache_scope,
),
)
return TestSetup(process, results_file_name=results_file_name)
async def export_virtualenv(request: _ExportVenvRequest,
python_setup: PythonSetup,
pex_pex: PexPEX) -> ExportResult:
if request.resolve:
interpreter_constraints = InterpreterConstraints(
python_setup.resolves_to_interpreter_constraints.get(
request.resolve, python_setup.interpreter_constraints))
else:
interpreter_constraints = InterpreterConstraints.create_from_targets(
request.root_python_targets,
python_setup) or InterpreterConstraints(
python_setup.interpreter_constraints)
min_interpreter = interpreter_constraints.snap_to_minimum(
python_setup.interpreter_universe)
if not min_interpreter:
err_msg = ((
f"The resolve '{request.resolve}' (from `[python].resolves`) has invalid interpreter "
f"constraints, which are set via `[python].resolves_to_interpreter_constraints`: "
f"{interpreter_constraints}. Could not determine the minimum compatible interpreter."
) if request.resolve else (
"The following interpreter constraints were computed for all the targets for which "
f"export was requested: {interpreter_constraints}. There is no python interpreter "
"compatible with these constraints. Please restrict the target set to one that shares "
"a compatible interpreter."))
raise ExportError(err_msg)
requirements_pex = await Get(
Pex,
RequirementsPexRequest(
(tgt.address for tgt in request.root_python_targets),
internal_only=True,
hardcoded_interpreter_constraints=min_interpreter,
),
)
# Get the full python version (including patch #), so we can use it as the venv name.
res = await Get(
ProcessResult,
PexProcess(
pex=requirements_pex,
description="Get interpreter version",
argv=[
"-c",
"import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))"
],
),
)
py_version = res.stdout.strip().decode()
dest = (os.path.join("python", "virtualenvs", path_safe(request.resolve))
if request.resolve else os.path.join("python", "virtualenv"))
merged_digest = await Get(
Digest, MergeDigests([pex_pex.digest, requirements_pex.digest]))
pex_pex_path = os.path.join("{digest_root}", pex_pex.exe)
return ExportResult(
f"virtualenv for the resolve '{request.resolve}' (using {min_interpreter})",
dest,
digest=merged_digest,
post_processing_cmds=[
PostProcessingCommand(
[
pex_pex_path,
os.path.join("{digest_root}", requirements_pex.name),
"venv",
"--pip",
"--collisions-ok",
"--remove=all",
f"{{digest_root}}/{py_version}",
],
{"PEX_MODULE": "pex.tools"},
),
PostProcessingCommand(["rm", "-f", pex_pex_path]),
],
)
async def export_virtualenv(request: _ExportVenvRequest,
python_setup: PythonSetup,
pex_pex: PexPEX) -> ExportResult:
if request.resolve:
interpreter_constraints = InterpreterConstraints(
python_setup.resolves_to_interpreter_constraints.get(
request.resolve, python_setup.interpreter_constraints))
else:
interpreter_constraints = InterpreterConstraints.create_from_targets(
request.root_python_targets,
python_setup) or InterpreterConstraints(
python_setup.interpreter_constraints)
requirements_pex = await Get(
Pex,
RequirementsPexRequest(
(tgt.address for tgt in request.root_python_targets),
hardcoded_interpreter_constraints=interpreter_constraints,
),
)
# Note that an internal-only pex will always have the `python` field set.
# See the build_pex() rule in pex.py.
interpreter = cast(PythonExecutable, requirements_pex.python)
# Get the full python version (including patch #), so we can use it as the venv name.
res = await Get(
ProcessResult,
Process(
description="Get interpreter version",
argv=[
interpreter.path,
"-c",
"import sys; print('.'.join(str(x) for x in sys.version_info[0:3]))",
],
),
)
py_version = res.stdout.strip().decode()
dest = (os.path.join("python", "virtualenvs", path_safe(request.resolve))
if request.resolve else os.path.join("python", "virtualenv"))
merged_digest = await Get(
Digest, MergeDigests([pex_pex.digest, requirements_pex.digest]))
pex_pex_path = os.path.join("{digest_root}", pex_pex.exe)
maybe_resolve_str = f"for the resolve '{request.resolve}' " if request.resolve else ""
return ExportResult(
f"virtualenv {maybe_resolve_str}(using Python {py_version})",
dest,
digest=merged_digest,
post_processing_cmds=[
PostProcessingCommand(
[
interpreter.path,
pex_pex_path,
os.path.join("{digest_root}", requirements_pex.name),
"venv",
"--pip",
"--collisions-ok",
"--remove=all",
f"{{digest_root}}/{py_version}",
],
{"PEX_MODULE": "pex.tools"},
),
PostProcessingCommand(["rm", "-f", pex_pex_path]),
],
)