async def generate_coverage_reports(
merged_coverage_data: MergedCoverageData,
coverage_setup: CoverageSetup,
coverage_config: CoverageConfig,
coverage_subsystem: CoverageSubsystem,
all_used_addresses: Addresses,
) -> CoverageReports:
"""Takes all Python test results and generates a single coverage report."""
transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest(all_used_addresses))
sources = await Get(
PythonSourceFiles,
# Coverage sometimes includes non-Python files in its `.coverage` data. We need to
# ensure that they're present when generating the report. We include all the files included
# by `pytest_runner.py`.
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True,
include_resources=True),
)
input_digest = await Get(
Digest,
MergeDigests((
merged_coverage_data.coverage_data,
coverage_config.digest,
sources.source_files.snapshot.digest,
)),
)
pex_processes = []
report_types = []
result_snapshot = await Get(Snapshot, Digest,
merged_coverage_data.coverage_data)
coverage_reports: List[CoverageReport] = []
for report_type in coverage_subsystem.reports:
if report_type == CoverageReportType.RAW:
coverage_reports.append(
FilesystemCoverageReport(
report_type=CoverageReportType.RAW.value,
result_snapshot=result_snapshot,
directory_to_materialize_to=coverage_subsystem.output_dir,
report_file=coverage_subsystem.output_dir / ".coverage",
))
continue
report_types.append(report_type)
output_file = (f"coverage.{report_type.value}" if report_type in {
CoverageReportType.XML, CoverageReportType.JSON
} else None)
pex_processes.append(
VenvPexProcess(
coverage_setup.pex,
argv=(report_type.report_name,
f"--rcfile={coverage_config.path}"),
input_digest=input_digest,
output_directories=("htmlcov", )
if report_type == CoverageReportType.HTML else None,
output_files=(output_file, ) if output_file else None,
description=
f"Generate Pytest {report_type.report_name} coverage report.",
level=LogLevel.DEBUG,
))
results = await MultiGet(
Get(ProcessResult, VenvPexProcess, process)
for process in pex_processes)
result_stdouts = tuple(res.stdout for res in results)
result_snapshots = await MultiGet(
Get(Snapshot, Digest, res.output_digest) for res in results)
coverage_reports.extend(
_get_coverage_report(coverage_subsystem.output_dir, report_type,
stdout, snapshot)
for (report_type, stdout,
snapshot) in zip(report_types, result_stdouts, result_snapshots))
return CoverageReports(tuple(coverage_reports))
async def pylint_lint_partition(partition: PylintPartition,
pylint: Pylint) -> LintResult:
requirements_pex_get = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(field_set.address for field_set in partition.field_sets),
# NB: These constraints must be identical to the other PEXes. Otherwise, we risk using
# a different version for the requirements than the other two PEXes, which can result
# in a PEX runtime error about missing dependencies.
hardcoded_interpreter_constraints=partition.
interpreter_constraints,
internal_only=True,
direct_deps_only=True,
),
)
plugin_requirements = PexRequirements.create_from_requirement_fields(
plugin_tgt[PythonRequirementsField]
for plugin_tgt in partition.plugin_targets
if plugin_tgt.has_field(PythonRequirementsField))
pylint_pex_get = Get(
Pex,
PexRequest(
output_filename="pylint.pex",
internal_only=True,
requirements=PexRequirements(
[*pylint.all_requirements, *plugin_requirements]),
interpreter_constraints=partition.interpreter_constraints,
),
)
prepare_plugin_sources_get = Get(
StrippedPythonSourceFiles,
PythonSourceFilesRequest(partition.plugin_targets))
prepare_python_sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(partition.targets_with_dependencies))
field_set_sources_get = Get(
SourceFiles,
SourceFilesRequest(field_set.sources
for field_set in partition.field_sets))
(
pylint_pex,
requirements_pex,
prepared_plugin_sources,
prepared_python_sources,
field_set_sources,
) = await MultiGet(
pylint_pex_get,
requirements_pex_get,
prepare_plugin_sources_get,
prepare_python_sources_get,
field_set_sources_get,
)
pylint_runner_pex, config_files = await MultiGet(
Get(
VenvPex,
PexRequest(
output_filename="pylint_runner.pex",
interpreter_constraints=partition.interpreter_constraints,
main=pylint.main,
internal_only=True,
pex_path=[pylint_pex, requirements_pex],
),
),
Get(ConfigFiles, ConfigFilesRequest,
pylint.config_request(field_set_sources.snapshot.dirs)),
)
prefixed_plugin_sources = (await Get(
Digest,
AddPrefix(
prepared_plugin_sources.stripped_source_files.snapshot.digest,
"__plugins"),
) if pylint.source_plugins else EMPTY_DIGEST)
pythonpath = list(prepared_python_sources.source_roots)
if pylint.source_plugins:
# NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must
# point to the plugin's directory, rather than to a parent's directory, because
# `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but
# not `path.to.plugin`. (This means users must have specified the parent directory as a
# source root.)
pythonpath.append("__plugins")
input_digest = await Get(
Digest,
MergeDigests((
config_files.snapshot.digest,
prefixed_plugin_sources,
prepared_python_sources.source_files.snapshot.digest,
)),
)
result = await Get(
FallibleProcessResult,
VenvPexProcess(
pylint_runner_pex,
argv=generate_argv(field_set_sources, pylint),
input_digest=input_digest,
extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},
description=
f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)))
async def mypy_typecheck_partition(partition: MyPyPartition, mypy: MyPy) -> TypecheckResult:
plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, mypy.source_plugins)
plugin_transitive_targets = await Get(
TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses)
)
plugin_requirements = PexRequirements.create_from_requirement_fields(
plugin_tgt[PythonRequirementsField]
for plugin_tgt in plugin_transitive_targets.closure
if plugin_tgt.has_field(PythonRequirementsField)
)
# If the user did not set `--python-version` already, we set it ourselves based on their code's
# interpreter constraints. This determines what AST is used by MyPy.
python_version = (
None
if partition.python_version_already_configured
else partition.interpreter_constraints.minimum_python_version()
)
# MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6,
# and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with
# Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a
# loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if
# 3.8+ is not installed on the machine.
tool_interpreter_constraints = (
partition.interpreter_constraints
if (
mypy.options.is_default("interpreter_constraints")
and partition.interpreter_constraints.requires_python38_or_newer()
)
else PexInterpreterConstraints(mypy.interpreter_constraints)
)
plugin_sources_request = Get(
PythonSourceFiles, PythonSourceFilesRequest(plugin_transitive_targets.closure)
)
typechecked_sources_request = Get(
PythonSourceFiles, PythonSourceFilesRequest(partition.closure)
)
requirements_pex_request = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(addr for addr in partition.field_set_addresses),
hardcoded_interpreter_constraints=partition.interpreter_constraints,
internal_only=True,
),
)
# TODO(John Sirois): Scope the extra requirements to the partition.
# Right now we just use a global set of extra requirements and these might not be compatible
# with all partitions. See: https://github.com/pantsbuild/pants/issues/11556
mypy_extra_requirements_pex_request = Get(
Pex,
PexRequest(
output_filename="mypy_extra_requirements.pex",
internal_only=True,
requirements=PexRequirements(mypy.extra_requirements),
interpreter_constraints=partition.interpreter_constraints,
),
)
mypy_pex_request = Get(
VenvPex,
PexRequest(
output_filename="mypy.pex",
internal_only=True,
main=mypy.main,
requirements=PexRequirements((*mypy.all_requirements, *plugin_requirements)),
interpreter_constraints=tool_interpreter_constraints,
),
)
config_digest_request = Get(Digest, PathGlobs, config_path_globs(mypy))
(
plugin_sources,
typechecked_sources,
mypy_pex,
requirements_pex,
mypy_extra_requirements_pex,
config_digest,
) = await MultiGet(
plugin_sources_request,
typechecked_sources_request,
mypy_pex_request,
requirements_pex_request,
mypy_extra_requirements_pex_request,
config_digest_request,
)
typechecked_srcs_snapshot = typechecked_sources.source_files.snapshot
file_list_path = "__files.txt"
python_files = "\n".join(
determine_python_files(typechecked_sources.source_files.snapshot.files)
)
file_list_digest_request = Get(
Digest,
CreateDigest([FileContent(file_list_path, python_files.encode())]),
)
typechecked_venv_pex_request = Get(
VenvPex,
PexRequest(
output_filename="typechecked_venv.pex",
internal_only=True,
pex_path=[requirements_pex, mypy_extra_requirements_pex],
interpreter_constraints=partition.interpreter_constraints,
),
)
typechecked_venv_pex, file_list_digest = await MultiGet(
typechecked_venv_pex_request, file_list_digest_request
)
merged_input_files = await Get(
Digest,
MergeDigests(
[
file_list_digest,
plugin_sources.source_files.snapshot.digest,
typechecked_srcs_snapshot.digest,
typechecked_venv_pex.digest,
config_digest,
]
),
)
all_used_source_roots = sorted(
set(itertools.chain(plugin_sources.source_roots, typechecked_sources.source_roots))
)
env = {
"PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots),
}
result = await Get(
FallibleProcessResult,
VenvPexProcess(
mypy_pex,
argv=generate_argv(
mypy,
typechecked_venv_pex=typechecked_venv_pex,
file_list_path=file_list_path,
python_version=python_version,
),
input_digest=merged_input_files,
extra_env=env,
description=f"Run MyPy on {pluralize(len(typechecked_srcs_snapshot.files), 'file')}.",
level=LogLevel.DEBUG,
),
)
return TypecheckResult.from_fallible_process_result(
result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints))
)
async def run_pep517_build(request: DistBuildRequest, python_setup: PythonSetup) -> DistBuildResult:
# Note that this pex has no entrypoint. We use it to run our generated shim, which
# in turn imports from and invokes the build backend.
build_backend_pex = await Get(
VenvPex,
PexRequest(
output_filename="build_backend.pex",
internal_only=True,
requirements=request.build_system.requires,
interpreter_constraints=request.interpreter_constraints,
),
)
dist_dir = "dist"
backend_shim_name = "backend_shim.py"
backend_shim_path = os.path.join(request.working_directory, backend_shim_name)
backend_shim_digest = await Get(
Digest,
CreateDigest(
[
FileContent(backend_shim_path, interpolate_backend_shim(dist_dir, request)),
]
),
)
merged_digest = await Get(Digest, MergeDigests((request.input, backend_shim_digest)))
if python_setup.macos_big_sur_compatibility and is_macos_big_sur():
extra_env = {"MACOSX_DEPLOYMENT_TARGET": "10.16"}
else:
extra_env = {}
result = await Get(
ProcessResult,
VenvPexProcess(
build_backend_pex,
argv=(backend_shim_name,),
input_digest=merged_digest,
extra_env=extra_env,
working_directory=request.working_directory,
output_directories=(dist_dir,), # Relative to the working_directory.
description=(
f"Run {request.build_system.build_backend} for {request.target_address_spec}"
if request.target_address_spec
else f"Run {request.build_system.build_backend}"
),
level=LogLevel.DEBUG,
),
)
output_lines = result.stdout.decode().splitlines()
paths = {}
for line in output_lines:
for dist_type in ["wheel", "sdist"]:
if line.startswith(f"{dist_type}: "):
paths[dist_type] = line[len(dist_type) + 2 :].strip()
# Note that output_digest paths are relative to the working_directory.
output_digest = await Get(Digest, RemovePrefix(result.output_digest, dist_dir))
output_snapshot = await Get(Snapshot, Digest, output_digest)
for dist_type, path in paths.items():
if path not in output_snapshot.files:
raise BuildBackendError(
f"Build backend {request.build_system.build_backend} did not create "
f"expected {dist_type} file {path}"
)
return DistBuildResult(
output_digest, wheel_path=paths.get("wheel"), sdist_path=paths.get("sdist")
)
async def package_python_google_cloud_function(
field_set: PythonGoogleCloudFunctionFieldSet,
lambdex: Lambdex,
union_membership: UnionMembership,
) -> BuiltPackage:
output_filename = field_set.output_path.value_or_default(
# Cloud Functions typically use the .zip suffix, so we use that instead of .pex.
file_ending="zip", )
# We hardcode the platform value to the appropriate one for each Google Cloud Function runtime.
# (Running the "hello world" cloud function in the example code will report the platform, and can be
# used to verify correctness of these platform strings.)
py_major, py_minor = field_set.runtime.to_interpreter_version()
platform = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}"
# set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707
if py_major <= 3 and py_minor < 8:
platform += "m"
additional_pex_args = (
# Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels.
"--manylinux=manylinux2014",
# When we're executing Pex on Linux, allow a local interpreter to be resolved if
# available and matching the AMI platform.
"--resolve-local-platforms",
)
pex_request = PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
output_filename=output_filename,
platforms=PexPlatforms([platform]),
additional_args=additional_pex_args,
additional_lockfile_args=additional_pex_args,
)
lambdex_request = PexRequest(
output_filename="lambdex.pex",
internal_only=True,
requirements=lambdex.pex_requirements(),
interpreter_constraints=lambdex.interpreter_constraints,
main=lambdex.main,
)
lambdex_pex, pex_result, handler, transitive_targets = await MultiGet(
Get(VenvPex, PexRequest, lambdex_request),
Get(Pex, PexFromTargetsRequest, pex_request),
Get(ResolvedPythonGoogleHandler,
ResolvePythonGoogleHandlerRequest(field_set.handler)),
Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])),
)
# Warn if users depend on `files` targets, which won't be included in the PEX and is a common
# gotcha.
files_tgts = targets_with_sources_types([FilesSources],
transitive_targets.dependencies,
union_membership)
if files_tgts:
files_addresses = sorted(tgt.address.spec for tgt in files_tgts)
logger.warning(
f"The python_google_cloud_function target {field_set.address} transitively depends on the below "
"files targets, but Pants will not include them in the built Cloud Function. Filesystem APIs "
"like `open()` are not able to load files within the binary itself; instead, they "
"read from the current working directory."
f"\n\nInstead, use `resources` targets. See {doc_url('resources')}."
f"\n\nFiles targets dependencies: {files_addresses}")
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
result = await Get(
ProcessResult,
VenvPexProcess(
lambdex_pex,
argv=("build", "-M", "main.py", "-e", handler.val,
output_filename),
input_digest=pex_result.digest,
output_files=(output_filename, ),
description=f"Setting up handler in {output_filename}",
),
)
artifact = BuiltPackageArtifact(
output_filename,
extra_log_lines=(
f" Runtime: {field_set.runtime.value}",
# The GCP-facing handler function is always main.handler, which is the
# wrapper injected by lambdex that manages invocation of the actual handler.
" Handler: main.handler",
),
)
return BuiltPackage(digest=result.output_digest, artifacts=(artifact, ))
async def flake8_lint_partition(partition: Flake8Partition, flake8: Flake8,
lint_subsystem: LintSubsystem) -> LintResult:
flake8_pex_request = Get(
VenvPex,
PexRequest(
output_filename="flake8.pex",
internal_only=True,
requirements=PexRequirements(flake8.all_requirements),
interpreter_constraints=partition.interpreter_constraints,
main=flake8.main,
),
)
config_digest_request = Get(
Digest,
PathGlobs(
globs=[flake8.config] if flake8.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--flake8-config`",
),
)
source_files_request = Get(
SourceFiles,
SourceFilesRequest(field_set.sources
for field_set in partition.field_sets))
flake8_pex, config_digest, source_files = await MultiGet(
flake8_pex_request, config_digest_request, source_files_request)
input_digest = await Get(
Digest, MergeDigests((source_files.snapshot.digest, config_digest)))
report_file_name = "flake8_report.txt" if lint_subsystem.reports_dir else None
result = await Get(
FallibleProcessResult,
VenvPexProcess(
flake8_pex,
argv=generate_args(source_files=source_files,
flake8=flake8,
report_file_name=report_file_name),
input_digest=input_digest,
output_files=(report_file_name, ) if report_file_name else None,
description=
f"Run Flake8 on {pluralize(len(partition.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = None
if report_file_name:
report_digest = await Get(
Digest,
DigestSubset(
result.output_digest,
PathGlobs(
[report_file_name],
glob_match_error_behavior=GlobMatchErrorBehavior.warn,
description_of_origin="Flake8 report file",
),
),
)
report = LintReport(report_file_name, report_digest)
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def setup_pytest_for_target(
request: TestSetupRequest,
pytest: PyTest,
test_subsystem: TestSubsystem,
python_setup: PythonSetup,
coverage_config: CoverageConfig,
coverage_subsystem: CoverageSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
) -> TestSetup:
transitive_targets, plugin_setups = await MultiGet(
Get(TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address])),
Get(AllPytestPluginSetups,
AllPytestPluginSetupsRequest(request.field_set.address)),
)
all_targets = transitive_targets.closure
interpreter_constraints = InterpreterConstraints.create_from_targets(
all_targets, python_setup)
requirements_pex_get = Get(
Pex,
RequirementsPexRequest([request.field_set.address],
internal_only=True))
pytest_pex_get = Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=pytest.pex_requirements(),
interpreter_constraints=interpreter_constraints,
internal_only=True,
),
)
# Ensure that the empty extra output dir exists.
extra_output_directory_digest_get = Get(
Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)]))
prepared_sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets, include_files=True))
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
field_set_source_files_get = Get(
SourceFiles, SourceFilesRequest([request.field_set.source]))
field_set_extra_env_get = Get(
Environment,
EnvironmentRequest(request.field_set.extra_env_vars.value or ()))
(
pytest_pex,
requirements_pex,
prepared_sources,
field_set_source_files,
field_set_extra_env,
extra_output_directory_digest,
) = await MultiGet(
pytest_pex_get,
requirements_pex_get,
prepared_sources_get,
field_set_source_files_get,
field_set_extra_env_get,
extra_output_directory_digest_get,
)
local_dists = await Get(
LocalDistsPex,
LocalDistsPexRequest(
[request.field_set.address],
internal_only=True,
interpreter_constraints=interpreter_constraints,
sources=prepared_sources,
),
)
pytest_runner_pex_get = Get(
VenvPex,
PexRequest(
output_filename="pytest_runner.pex",
interpreter_constraints=interpreter_constraints,
main=pytest.main,
internal_only=True,
pex_path=[pytest_pex, requirements_pex, local_dists.pex],
),
)
config_files_get = Get(
ConfigFiles,
ConfigFilesRequest,
pytest.config_request(field_set_source_files.snapshot.dirs),
)
pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get,
config_files_get)
# The coverage and pytest config may live in the same config file (e.g., setup.cfg, tox.ini
# or pyproject.toml), and wee may have rewritten those files to augment the coverage config,
# in which case we must ensure that the original and rewritten files don't collide.
pytest_config_digest = config_files.snapshot.digest
if coverage_config.path in config_files.snapshot.files:
subset_paths = list(config_files.snapshot.files)
# Remove the original file, and rely on the rewritten file, which contains all the
# pytest-related config unchanged.
subset_paths.remove(coverage_config.path)
pytest_config_digest = await Get(
Digest, DigestSubset(pytest_config_digest,
PathGlobs(subset_paths)))
input_digest = await Get(
Digest,
MergeDigests((
coverage_config.digest,
local_dists.remaining_sources.source_files.snapshot.digest,
pytest_config_digest,
extra_output_directory_digest,
*(plugin_setup.digest for plugin_setup in plugin_setups),
)),
)
add_opts = [f"--color={'yes' if global_options.colors else 'no'}"]
output_files = []
results_file_name = None
if not request.is_debug:
results_file_name = f"{request.field_set.address.path_safe_spec}.xml"
add_opts.extend((f"--junitxml={results_file_name}", "-o",
f"junit_family={pytest.junit_family}"))
output_files.append(results_file_name)
coverage_args = []
if test_subsystem.use_coverage and not request.is_debug:
pytest.validate_pytest_cov_included()
output_files.append(".coverage")
if coverage_subsystem.filter:
cov_args = [f"--cov={morf}" for morf in coverage_subsystem.filter]
else:
# N.B.: Passing `--cov=` or `--cov=.` to communicate "record coverage for all sources"
# fails in certain contexts as detailed in:
# https://github.com/pantsbuild/pants/issues/12390
# Instead we focus coverage on just the directories containing python source files
# materialized to the Process chroot.
cov_args = [
f"--cov={source_root}"
for source_root in prepared_sources.source_roots
]
coverage_args = [
"--cov-report=", # Turn off output.
f"--cov-config={coverage_config.path}",
*cov_args,
]
extra_env = {
"PYTEST_ADDOPTS": " ".join(add_opts),
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
**test_extra_env.env,
# NOTE: field_set_extra_env intentionally after `test_extra_env` to allow overriding within
# `python_tests`.
**field_set_extra_env,
}
# Cache test runs only if they are successful, or not at all if `--test-force`.
cache_scope = (ProcessCacheScope.PER_SESSION
if test_subsystem.force else ProcessCacheScope.SUCCESSFUL)
process = await Get(
Process,
VenvPexProcess(
pytest_runner_pex,
argv=(*pytest.args, *coverage_args, *field_set_source_files.files),
extra_env=extra_env,
input_digest=input_digest,
output_directories=(_EXTRA_OUTPUT_DIR, ),
output_files=output_files,
timeout_seconds=request.field_set.timeout.
calculate_from_global_options(pytest),
execution_slot_variable=pytest.execution_slot_var,
description=f"Run Pytest for {request.field_set.address}",
level=LogLevel.DEBUG,
cache_scope=cache_scope,
),
)
return TestSetup(process, results_file_name=results_file_name)
def create_pex_and_get_all_data(
rule_runner: RuleRunner,
*,
pex_type: type[Pex | VenvPex] = Pex,
requirements: PexRequirements | Lockfile
| LockfileContent = PexRequirements(),
main: MainSpecification | None = None,
interpreter_constraints: InterpreterConstraints = InterpreterConstraints(),
platforms: PexPlatforms = PexPlatforms(),
sources: Digest | None = None,
additional_inputs: Digest | None = None,
additional_pants_args: tuple[str, ...] = (),
additional_pex_args: tuple[str, ...] = (),
env: Mapping[str, str] | None = None,
internal_only: bool = True,
) -> PexData:
request = PexRequest(
output_filename="test.pex",
internal_only=internal_only,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
platforms=platforms,
main=main,
sources=sources,
additional_inputs=additional_inputs,
additional_args=additional_pex_args,
)
rule_runner.set_options(
["--backend-packages=pants.backend.python", *additional_pants_args],
env=env,
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
)
pex: Pex | VenvPex
if pex_type == Pex:
pex = rule_runner.request(Pex, [request])
digest = pex.digest
sandbox_path = pex.name
pex_pex = rule_runner.request(PexPEX, [])
process = rule_runner.request(
Process,
[
PexProcess(
Pex(digest=pex_pex.digest,
name=pex_pex.exe,
python=pex.python),
argv=["-m", "pex.tools", pex.name, "info"],
input_digest=pex.digest,
extra_env=dict(PEX_INTERPRETER="1"),
description="Extract PEX-INFO.",
)
],
)
else:
pex = rule_runner.request(VenvPex, [request])
digest = pex.digest
sandbox_path = pex.pex_filename
process = rule_runner.request(
Process,
[
VenvPexProcess(
pex,
argv=["info"],
extra_env=dict(PEX_TOOLS="1"),
description="Extract PEX-INFO.",
),
],
)
rule_runner.scheduler.write_digest(digest)
local_path = PurePath(rule_runner.build_root) / "test.pex"
result = rule_runner.request(ProcessResult, [process])
pex_info_content = result.stdout.decode()
is_zipapp = zipfile.is_zipfile(local_path)
if is_zipapp:
with zipfile.ZipFile(local_path, "r") as zipfp:
files = tuple(zipfp.namelist())
else:
files = tuple(
os.path.normpath(
os.path.relpath(os.path.join(root, path), local_path))
for root, dirs, files in os.walk(local_path)
for path in dirs + files)
return PexData(
pex=pex,
is_zipapp=is_zipapp,
sandbox_path=PurePath(sandbox_path),
local_path=local_path,
info=json.loads(pex_info_content),
files=files,
)
async def pylint_lint_partition(
partition: PylintPartition, pylint: Pylint,
first_party_plugins: PylintFirstPartyPlugins) -> LintResult:
requirements_pex_get = Get(
Pex,
RequirementsPexRequest(
(fs.address for fs in partition.root_field_sets),
# NB: These constraints must be identical to the other PEXes. Otherwise, we risk using
# a different version for the requirements than the other two PEXes, which can result
# in a PEX runtime error about missing dependencies.
hardcoded_interpreter_constraints=partition.
interpreter_constraints,
),
)
pylint_pex_get = Get(
Pex,
PexRequest,
pylint.to_pex_request(
interpreter_constraints=partition.interpreter_constraints,
extra_requirements=first_party_plugins.requirement_strings,
),
)
prepare_python_sources_get = Get(
PythonSourceFiles, PythonSourceFilesRequest(partition.closure))
field_set_sources_get = Get(
SourceFiles,
SourceFilesRequest(fs.source for fs in partition.root_field_sets))
# Ensure that the empty report dir exists.
report_directory_digest_get = Get(Digest,
CreateDigest([Directory(REPORT_DIR)]))
(
pylint_pex,
requirements_pex,
prepared_python_sources,
field_set_sources,
report_directory,
) = await MultiGet(
pylint_pex_get,
requirements_pex_get,
prepare_python_sources_get,
field_set_sources_get,
report_directory_digest_get,
)
pylint_runner_pex, config_files = await MultiGet(
Get(
VenvPex,
VenvPexRequest(
PexRequest(
output_filename="pylint_runner.pex",
interpreter_constraints=partition.interpreter_constraints,
main=pylint.main,
internal_only=True,
pex_path=[pylint_pex, requirements_pex],
),
# TODO(John Sirois): Remove this (change to the default of symlinks) when we can
# upgrade to a version of Pylint with https://github.com/PyCQA/pylint/issues/1470
# resolved.
site_packages_copies=True,
),
),
Get(ConfigFiles, ConfigFilesRequest,
pylint.config_request(field_set_sources.snapshot.dirs)),
)
pythonpath = list(prepared_python_sources.source_roots)
if first_party_plugins:
pythonpath.append(first_party_plugins.PREFIX)
input_digest = await Get(
Digest,
MergeDigests((
config_files.snapshot.digest,
first_party_plugins.sources_digest,
prepared_python_sources.source_files.snapshot.digest,
report_directory,
)),
)
result = await Get(
FallibleProcessResult,
VenvPexProcess(
pylint_runner_pex,
argv=generate_argv(field_set_sources, pylint),
input_digest=input_digest,
output_directories=(REPORT_DIR, ),
extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},
concurrency_available=len(partition.root_field_sets),
description=
f"Run Pylint on {pluralize(len(partition.root_field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR))
return LintResult.from_fallible_process_result(
result,
partition_description=partition.description(),
report=report,
)
async def twine_upload(
request: PublishPythonPackageRequest,
twine_subsystem: TwineSubsystem,
global_options: GlobalOptions,
) -> PublishProcesses:
dists = tuple(artifact.relpath for pkg in request.packages
for artifact in pkg.artifacts if artifact.relpath)
if twine_subsystem.skip or not dists:
return PublishProcesses()
# Too verbose to provide feedback as to why some packages were skipped?
skip = None
if request.field_set.skip_twine.value:
skip = f"(by `{request.field_set.skip_twine.alias}` on {request.field_set.address})"
elif not request.field_set.repositories.value:
# I'd rather have used the opt_out mechanism on the field set, but that gives no hint as to
# why the target was not applicable..
skip = f"(no `{request.field_set.repositories.alias}` specified for {request.field_set.address})"
if skip:
return PublishProcesses([
PublishPackages(
names=dists,
description=skip,
),
])
twine_pex, packages_digest, config_files = await MultiGet(
Get(VenvPex, PexRequest, twine_subsystem.to_pex_request()),
Get(Digest, MergeDigests(pkg.digest for pkg in request.packages)),
Get(ConfigFiles, ConfigFilesRequest, twine_subsystem.config_request()),
)
ca_cert_request = twine_subsystem.ca_certs_digest_request(
global_options.ca_certs_path)
ca_cert = await Get(Snapshot, CreateDigest,
ca_cert_request) if ca_cert_request else None
ca_cert_digest = (ca_cert.digest, ) if ca_cert else ()
input_digest = await Get(
Digest,
MergeDigests(
(packages_digest, config_files.snapshot.digest, *ca_cert_digest)))
pex_proc_requests = []
twine_envs = await MultiGet(
Get(Environment, EnvironmentRequest, twine_env_request(repo))
for repo in request.field_set.repositories.value)
for repo, env in zip(request.field_set.repositories.value, twine_envs):
pex_proc_requests.append(
VenvPexProcess(
twine_pex,
argv=twine_upload_args(twine_subsystem, config_files, repo,
dists, ca_cert),
input_digest=input_digest,
extra_env=twine_env(env, repo),
description=repo,
))
processes = await MultiGet(
Get(Process, VenvPexProcess, request) for request in pex_proc_requests)
return PublishProcesses(
PublishPackages(
names=dists,
process=InteractiveProcess.from_process(process),
description=process.description,
data=PublishOutputData({"repository": process.description}),
) for process in processes)
async def setup_pytest_for_target(
request: TestSetupRequest,
pytest: PyTest,
test_subsystem: TestSubsystem,
python_setup: PythonSetup,
coverage_config: CoverageConfig,
coverage_subsystem: CoverageSubsystem,
test_extra_env: TestExtraEnv,
global_options: GlobalOptions,
) -> TestSetup:
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.field_set.address]))
all_targets = transitive_targets.closure
interpreter_constraints = PexInterpreterConstraints.create_from_targets(
all_targets, python_setup)
requirements_pex_get = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([request.field_set.address],
internal_only=True),
)
pytest_pex_get = Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
interpreter_constraints=interpreter_constraints,
internal_only=True,
),
)
config_files_get = Get(ConfigFiles, ConfigFilesRequest,
pytest.config_request)
extra_output_directory_digest_get = Get(
Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)]))
prepared_sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets, include_files=True))
# Create any assets that the test depends on through the `runtime_package_dependencies` field.
assets: Tuple[BuiltPackage, ...] = ()
unparsed_runtime_packages = (request.field_set.runtime_package_dependencies
.to_unparsed_address_inputs())
if unparsed_runtime_packages.values:
runtime_package_targets = await Get(Targets, UnparsedAddressInputs,
unparsed_runtime_packages)
field_sets_per_target = await Get(
FieldSetsPerTarget,
FieldSetsPerTargetRequest(PackageFieldSet,
runtime_package_targets),
)
assets = await MultiGet(
Get(BuiltPackage, PackageFieldSet, field_set)
for field_set in field_sets_per_target.field_sets)
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
field_set_source_files_get = Get(
SourceFiles, SourceFilesRequest([request.field_set.sources]))
(
pytest_pex,
requirements_pex,
prepared_sources,
field_set_source_files,
config_files,
extra_output_directory_digest,
) = await MultiGet(
pytest_pex_get,
requirements_pex_get,
prepared_sources_get,
field_set_source_files_get,
config_files_get,
extra_output_directory_digest_get,
)
pytest_runner_pex = await Get(
VenvPex,
PexRequest(
output_filename="pytest_runner.pex",
interpreter_constraints=interpreter_constraints,
main=ConsoleScript("pytest"),
internal_only=True,
pex_path=[pytest_pex, requirements_pex],
),
)
input_digest = await Get(
Digest,
MergeDigests((
coverage_config.digest,
prepared_sources.source_files.snapshot.digest,
config_files.snapshot.digest,
extra_output_directory_digest,
*(binary.digest for binary in assets),
)),
)
add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"]
output_files = []
results_file_name = None
if pytest.options.junit_xml_dir and not request.is_debug:
results_file_name = f"{request.field_set.address.path_safe_spec}.xml"
add_opts.extend((f"--junitxml={results_file_name}", "-o",
f"junit_family={pytest.options.junit_family}"))
output_files.append(results_file_name)
coverage_args = []
if test_subsystem.use_coverage and not request.is_debug:
output_files.append(".coverage")
cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else (
".", )
coverage_args = [
"--cov-report=", # Turn off output.
*itertools.chain.from_iterable(["--cov", cov_path]
for cov_path in cov_paths),
]
extra_env = {
"PYTEST_ADDOPTS": " ".join(add_opts),
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
**test_extra_env.env,
}
# Cache test runs only if they are successful, or not at all if `--test-force`.
cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL
process = await Get(
Process,
VenvPexProcess(
pytest_runner_pex,
argv=(*pytest.options.args, *coverage_args,
*field_set_source_files.files),
extra_env=extra_env,
input_digest=input_digest,
output_directories=(_EXTRA_OUTPUT_DIR, ),
output_files=output_files,
timeout_seconds=request.field_set.timeout.
calculate_from_global_options(pytest),
execution_slot_variable=pytest.options.execution_slot_var,
description=f"Run Pytest for {request.field_set.address}",
level=LogLevel.DEBUG,
cache_scope=cache_scope,
),
)
return TestSetup(process, results_file_name=results_file_name)
async def generate_python_from_setuptools_scm(
request: GeneratePythonFromSetuptoolsSCMRequest,
setuptools_scm: SetuptoolsSCM,
) -> GeneratedSources:
# A GitWorktreeRequest is uncacheable, so this enclosing rule will run every time its result
# is needed, meaning it will always return a result based on the current underlying git state.
maybe_git_worktree = await Get(MaybeGitWorktree, GitWorktreeRequest())
if not maybe_git_worktree.git_worktree:
raise VCSVersioningError(
softwrap(
f"""
Trying to determine the version for the {request.protocol_target.address} target at
{request.protocol_target.address}, but you are not running in a git worktree.
"""
)
)
# Generate the setuptools_scm config. We don't use any existing pyproject.toml config,
# because we don't want to let setuptools_scm itself write the output file. This is because
# it would do so relative to the --root, meaning it will write outside the sandbox,
# directly into the workspace, which is obviously not what we want.
# It's unfortunate that setuptools_scm does not have separate config for "where is the .git
# directory" and "where should I write output to".
config: dict[str, dict[str, dict[str, str]]] = {}
tool_config = config.setdefault("tool", {}).setdefault("setuptools_scm", {})
tag_regex = request.protocol_target[VersionTagRegexField].value
if tag_regex:
tool_config["tag_regex"] = tag_regex
config_path = "pyproject.synthetic.toml"
input_digest_get = Get(
Digest,
CreateDigest(
[
FileContent(config_path, toml.dumps(config).encode()),
]
),
)
setuptools_scm_pex_get = Get(VenvPex, PexRequest, setuptools_scm.to_pex_request())
setuptools_scm_pex, input_digest = await MultiGet(setuptools_scm_pex_get, input_digest_get)
argv = ["--root", str(maybe_git_worktree.git_worktree.worktree), "--config", config_path]
result = await Get(
ProcessResult,
VenvPexProcess(
setuptools_scm_pex,
argv=argv,
input_digest=input_digest,
description=f"Run setuptools_scm for {request.protocol_target.address.spec}",
level=LogLevel.INFO,
),
)
version = result.stdout.decode().strip()
write_to = cast(str, request.protocol_target[VersionGenerateToField].value)
write_to_template = cast(str, request.protocol_target[VersionTemplateField].value)
output_content = write_to_template.format(version=version)
output_snapshot = await Get(
Snapshot, CreateDigest([FileContent(write_to, output_content.encode())])
)
return GeneratedSources(output_snapshot)
async def mypy_typecheck_partition(
partition: MyPyPartition,
config_file: MyPyConfigFile,
first_party_plugins: MyPyFirstPartyPlugins,
mypy: MyPy,
python_setup: PythonSetup,
) -> CheckResult:
# MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6,
# and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with
# Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a
# loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if
# 3.8+ is not installed on the machine.
tool_interpreter_constraints = (
partition.interpreter_constraints
if (
mypy.options.is_default("interpreter_constraints")
and partition.interpreter_constraints.requires_python38_or_newer(
python_setup.interpreter_universe
)
)
else mypy.interpreter_constraints
)
closure_sources_get = Get(PythonSourceFiles, PythonSourceFilesRequest(partition.closure))
roots_sources_get = Get(
SourceFiles,
SourceFilesRequest(fs.sources for fs in partition.root_field_sets),
)
# See `requirements_venv_pex` for how this will get wrapped in a `VenvPex`.
requirements_pex_get = Get(
Pex,
RequirementsPexRequest(
(fs.address for fs in partition.root_field_sets),
hardcoded_interpreter_constraints=partition.interpreter_constraints,
),
)
extra_type_stubs_pex_get = Get(
Pex,
PexRequest(
output_filename="extra_type_stubs.pex",
internal_only=True,
requirements=PexRequirements(mypy.extra_type_stubs),
interpreter_constraints=partition.interpreter_constraints,
),
)
mypy_pex_get = Get(
VenvPex,
PexRequest,
mypy.to_pex_request(
interpreter_constraints=tool_interpreter_constraints,
extra_requirements=first_party_plugins.requirement_strings,
),
)
(
closure_sources,
roots_sources,
mypy_pex,
extra_type_stubs_pex,
requirements_pex,
) = await MultiGet(
closure_sources_get,
roots_sources_get,
mypy_pex_get,
extra_type_stubs_pex_get,
requirements_pex_get,
)
python_files = determine_python_files(roots_sources.snapshot.files)
file_list_path = "__files.txt"
file_list_digest_request = Get(
Digest,
CreateDigest([FileContent(file_list_path, "\n".join(python_files).encode())]),
)
# This creates a venv with all the 3rd-party requirements used by the code. We tell MyPy to
# use this venv by setting `--python-executable`. Note that this Python interpreter is
# different than what we run MyPy with.
#
# We could have directly asked the `PexFromTargetsRequest` to return a `VenvPex`, rather than
# `Pex`, but that would mean missing out on sharing a cache with other goals like `test` and
# `run`.
requirements_venv_pex_request = Get(
VenvPex,
PexRequest(
output_filename="requirements_venv.pex",
internal_only=True,
pex_path=[requirements_pex, extra_type_stubs_pex],
interpreter_constraints=partition.interpreter_constraints,
),
)
requirements_venv_pex, file_list_digest = await MultiGet(
requirements_venv_pex_request, file_list_digest_request
)
merged_input_files = await Get(
Digest,
MergeDigests(
[
file_list_digest,
first_party_plugins.sources_digest,
closure_sources.source_files.snapshot.digest,
requirements_venv_pex.digest,
config_file.digest,
]
),
)
all_used_source_roots = sorted(
set(itertools.chain(first_party_plugins.source_roots, closure_sources.source_roots))
)
env = {
"PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots),
"MYPYPATH": ":".join(all_used_source_roots),
}
result = await Get(
FallibleProcessResult,
VenvPexProcess(
mypy_pex,
argv=generate_argv(
mypy,
venv_python=requirements_venv_pex.python.argv0,
file_list_path=file_list_path,
python_version=config_file.python_version_to_autoset(
partition.interpreter_constraints, python_setup.interpreter_universe
),
),
input_digest=merged_input_files,
extra_env=env,
output_directories=(REPORT_DIR,),
description=f"Run MyPy on {pluralize(len(python_files), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR))
return CheckResult.from_fallible_process_result(
result,
partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def bandit_lint_partition(partition: BanditPartition, bandit: Bandit,
lint_subsystem: LintSubsystem) -> LintResult:
bandit_pex_get = Get(
VenvPex,
PexRequest(
output_filename="bandit.pex",
internal_only=True,
requirements=PexRequirements(bandit.all_requirements),
interpreter_constraints=partition.interpreter_constraints,
main=bandit.main,
),
)
config_files_get = Get(ConfigFiles, ConfigFilesRequest,
bandit.config_request)
source_files_get = Get(
SourceFiles,
SourceFilesRequest(field_set.sources
for field_set in partition.field_sets))
bandit_pex, config_files, source_files = await MultiGet(
bandit_pex_get, config_files_get, source_files_get)
input_digest = await Get(
Digest,
MergeDigests(
(source_files.snapshot.digest, config_files.snapshot.digest)))
report_file_name = "bandit_report.txt" if lint_subsystem.reports_dir else None
result = await Get(
FallibleProcessResult,
VenvPexProcess(
bandit_pex,
argv=generate_args(source_files=source_files,
bandit=bandit,
report_file_name=report_file_name),
input_digest=input_digest,
description=
f"Run Bandit on {pluralize(len(partition.field_sets), 'file')}.",
output_files=(report_file_name, ) if report_file_name else None,
level=LogLevel.DEBUG,
),
)
report = None
if report_file_name:
report_digest = await Get(
Digest,
DigestSubset(
result.output_digest,
PathGlobs(
[report_file_name],
glob_match_error_behavior=GlobMatchErrorBehavior.warn,
description_of_origin="Bandit report file",
),
),
)
report = LintReport(report_file_name, report_digest)
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def run_setup_py(req: RunSetupPyRequest, setuptools: Setuptools,
python_setup: PythonSetup) -> RunSetupPyResult:
"""Run a setup.py command on a single exported target."""
# Note that this pex has no entrypoint. We use it to run our generated setup.py, which
# in turn imports from and invokes setuptools.
setuptools_pex = await Get(
VenvPex,
PexRequest(
output_filename="setuptools.pex",
internal_only=True,
requirements=setuptools.pex_requirements(),
interpreter_constraints=req.interpreter_constraints,
),
)
# We prefix the entire chroot, and run with this prefix as the cwd, so that we can capture any
# changes setup made within it (e.g., when running 'develop') without also capturing other
# artifacts of the pex process invocation.
chroot_prefix = "chroot"
# The setuptools dist dir, created by it under the chroot (not to be confused with
# pants's own dist dir, at the buildroot).
dist_dir = "dist"
prefixed_chroot = await Get(Digest,
AddPrefix(req.chroot.digest, chroot_prefix))
# setup.py basically always expects to be run with the cwd as its own directory
# (e.g., paths in it are relative to that directory). This is true of the setup.py
# we generate and is overwhelmingly likely to be true for existing setup.py files,
# as there is no robust way to run them otherwise.
setup_script_reldir, setup_script_name = os.path.split(
req.chroot.setup_script)
working_directory = os.path.join(chroot_prefix, setup_script_reldir)
if python_setup.macos_big_sur_compatibility and is_macos_big_sur():
extra_env = {"MACOSX_DEPLOYMENT_TARGET": "10.16"}
else:
extra_env = {}
result = await Get(
ProcessResult,
VenvPexProcess(
setuptools_pex,
argv=(setup_script_name, *req.args),
input_digest=prefixed_chroot,
extra_env=extra_env,
working_directory=working_directory,
# setuptools commands that create dists write them to the distdir.
# TODO: Could there be other useful files to capture?
output_directories=(
dist_dir, ), # Relative to the working_directory.
description=
f"Run setuptools for {req.exported_target.target.address}",
level=LogLevel.DEBUG,
),
)
# Note that output_digest paths are relative to the working_directory.
output_digest = await Get(Digest,
RemovePrefix(result.output_digest, dist_dir))
return RunSetupPyResult(output_digest)
async def generate_lockfile(
req: GeneratePythonLockfile,
poetry_subsystem: PoetrySubsystem,
generate_lockfiles_subsystem: GenerateLockfilesSubsystem,
python_repos: PythonRepos,
python_setup: PythonSetup,
) -> GenerateLockfileResult:
if req.use_pex:
pip_args_file = "__pip_args.txt"
pip_args_file_content = "\n".join(
[f"--no-binary {pkg}" for pkg in python_setup.no_binary] +
[f"--only-binary {pkg}" for pkg in python_setup.only_binary])
pip_args_file_digest = await Get(
Digest,
CreateDigest(
[FileContent(pip_args_file, pip_args_file_content.encode())]))
header_delimiter = "//"
result = await Get(
ProcessResult,
PexCliProcess(
subcommand=("lock", "create"),
extra_args=(
"--output=lock.json",
"--no-emit-warnings",
# See https://github.com/pantsbuild/pants/issues/12458. For now, we always
# generate universal locks because they have the best compatibility. We may
# want to let users change this, as `style=strict` is safer.
"--style=universal",
"--resolver-version",
"pip-2020-resolver",
# This makes diffs more readable when lockfiles change.
"--indent=2",
"-r",
pip_args_file,
*python_repos.pex_args,
*python_setup.manylinux_pex_args,
*req.interpreter_constraints.generate_pex_arg_list(),
*req.requirements,
),
additional_input_digest=pip_args_file_digest,
output_files=("lock.json", ),
description=f"Generate lockfile for {req.resolve_name}",
# Instead of caching lockfile generation with LMDB, we instead use the invalidation
# scheme from `lockfile_metadata.py` to check for stale/invalid lockfiles. This is
# necessary so that our invalidation is resilient to deleting LMDB or running on a
# new machine.
#
# We disable caching with LMDB so that when you generate a lockfile, you always get
# the most up-to-date snapshot of the world. This is generally desirable and also
# necessary to avoid an awkward edge case where different developers generate
# different lockfiles even when generating at the same time. See
# https://github.com/pantsbuild/pants/issues/12591.
cache_scope=ProcessCacheScope.PER_SESSION,
),
)
else:
header_delimiter = "#"
await Get(MaybeWarnPythonRepos, MaybeWarnPythonReposRequest())
_pyproject_toml = create_pyproject_toml(
req.requirements, req.interpreter_constraints).encode()
_pyproject_toml_digest, _launcher_digest = await MultiGet(
Get(Digest,
CreateDigest([FileContent("pyproject.toml",
_pyproject_toml)])),
Get(Digest, CreateDigest([POETRY_LAUNCHER])),
)
_poetry_pex = await Get(
VenvPex,
PexRequest,
poetry_subsystem.to_pex_request(main=EntryPoint(
PurePath(POETRY_LAUNCHER.path).stem),
sources=_launcher_digest),
)
# WONTFIX(#12314): Wire up Poetry to named_caches.
# WONTFIX(#12314): Wire up all the pip options like indexes.
_lock_result = await Get(
ProcessResult,
VenvPexProcess(
_poetry_pex,
argv=("lock", ),
input_digest=_pyproject_toml_digest,
output_files=("poetry.lock", "pyproject.toml"),
description=f"Generate lockfile for {req.resolve_name}",
cache_scope=ProcessCacheScope.PER_SESSION,
),
)
result = await Get(
ProcessResult,
VenvPexProcess(
_poetry_pex,
argv=("export", "-o", req.lockfile_dest),
input_digest=_lock_result.output_digest,
output_files=(req.lockfile_dest, ),
description=
(f"Exporting Poetry lockfile to requirements.txt format for {req.resolve_name}"
),
level=LogLevel.DEBUG,
),
)
initial_lockfile_digest_contents = await Get(DigestContents, Digest,
result.output_digest)
# TODO(#12314) Improve error message on `Requirement.parse`
metadata = PythonLockfileMetadata.new(
req.interpreter_constraints,
{PipRequirement.parse(i)
for i in req.requirements},
)
lockfile_with_header = metadata.add_header_to_lockfile(
initial_lockfile_digest_contents[0].content,
regenerate_command=(
generate_lockfiles_subsystem.custom_command or
f"{bin_name()} generate-lockfiles --resolve={req.resolve_name}"),
delimeter=header_delimiter,
)
final_lockfile_digest = await Get(
Digest,
CreateDigest([FileContent(req.lockfile_dest, lockfile_with_header)]))
return GenerateLockfileResult(final_lockfile_digest, req.resolve_name,
req.lockfile_dest)
async def setup_black(setup_request: SetupRequest, black: Black,
python_setup: PythonSetup) -> Setup:
# Black requires 3.6+ but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6, and 3.7.
# However, typed-ast does not understand 3.8+, so instead we must run Black with Python 3.8+
# when relevant. We only do this if if <3.8 can't be used, as we don't want a loose requirement
# like `>=3.6` to result in requiring Python 3.8, which would error if 3.8 is not installed on
# the machine.
all_interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(field_set.interpreter_constraints
for field_set in setup_request.request.field_sets),
python_setup,
)
tool_interpreter_constraints = (
all_interpreter_constraints if
(all_interpreter_constraints.requires_python38_or_newer()
and black.options.is_default("interpreter_constraints")) else
PexInterpreterConstraints(black.interpreter_constraints))
black_pex_get = Get(
VenvPex,
PexRequest(
output_filename="black.pex",
internal_only=True,
requirements=PexRequirements(black.all_requirements),
interpreter_constraints=tool_interpreter_constraints,
main=black.main,
),
)
source_files_get = Get(
SourceFiles,
SourceFilesRequest(field_set.sources
for field_set in setup_request.request.field_sets),
)
source_files, black_pex = await MultiGet(source_files_get, black_pex_get)
source_files_snapshot = (
source_files.snapshot
if setup_request.request.prior_formatter_result is None else
setup_request.request.prior_formatter_result)
config_files = await Get(ConfigFiles, ConfigFilesRequest,
black.config_request(source_files_snapshot.dirs))
input_digest = await Get(
Digest,
MergeDigests(
(source_files_snapshot.digest, config_files.snapshot.digest)))
process = await Get(
Process,
VenvPexProcess(
black_pex,
argv=generate_argv(source_files,
black,
check_only=setup_request.check_only),
input_digest=input_digest,
output_files=source_files_snapshot.files,
description=
f"Run Black on {pluralize(len(setup_request.request.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
return Setup(process, original_digest=source_files_snapshot.digest)
