async def flake8_lint_partition(
partition: Flake8Partition,
flake8: Flake8,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
requirements_pex_request = Get[Pex](PexRequest(
output_filename="flake8.pex",
requirements=PexRequirements(flake8.get_requirement_specs()),
interpreter_constraints=(partition.interpreter_constraints
or PexInterpreterConstraints(
flake8.default_interpreter_constraints)),
entry_point=flake8.get_entry_point(),
))
config_path: Optional[str] = flake8.options.config
config_snapshot_request = Get[Snapshot](PathGlobs(
globs=[config_path] if config_path else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--flake8-config`",
))
all_source_files_request = Get[SourceFiles](AllSourceFilesRequest(
field_set.sources for field_set in partition.field_sets))
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest((field_set.sources, field_set.origin)
for field_set in partition.field_sets))
requirements_pex, config_snapshot, all_source_files, specified_source_files = cast(
Tuple[Pex, Snapshot, SourceFiles, SourceFiles],
await MultiGet([
requirements_pex_request,
config_snapshot_request,
all_source_files_request,
specified_source_files_request,
]),
)
input_digest = await Get[Digest](MergeDigests(
(all_source_files.snapshot.digest, requirements_pex.digest,
config_snapshot.digest)))
address_references = ", ".join(
sorted(field_set.address.reference()
for field_set in partition.field_sets))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./flake8.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
flake8=flake8),
input_digest=input_digest,
description=
(f"Run Flake8 on {pluralize(len(partition.field_sets), 'target')}: {address_references}."
),
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result,
linter_name="Flake8")
async def setup(
request: SetupRequest,
isort: Isort,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
requirements_pex = await Get[Pex](PexRequest(
output_filename="isort.pex",
requirements=PexRequirements(isort.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
isort.default_interpreter_constraints),
entry_point=isort.get_entry_point(),
))
config_path: Optional[List[str]] = isort.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=config_path or (),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin="the option `--isort-config`",
))
if request.configs.prior_formatter_result is None:
all_source_files = await Get[SourceFiles](AllSourceFilesRequest(
config.sources for config in request.configs))
all_source_files_snapshot = all_source_files.snapshot
else:
all_source_files_snapshot = request.configs.prior_formatter_result
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest(
(config.sources, config.origin) for config in request.configs))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files_snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
address_references = ", ".join(
sorted(config.address.reference() for config in request.configs))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./isort.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
isort=isort,
check_only=request.check_only,
),
input_files=merged_input_files,
output_files=all_source_files_snapshot.files,
description=
(f"Run isort on {pluralize(len(request.configs), 'target')}: {address_references}."
),
)
return Setup(process)
async def setup(
setup_request: SetupRequest,
docformatter: Docformatter,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
requirements_pex_request = Get[Pex](PexRequest(
output_filename="docformatter.pex",
requirements=PexRequirements(docformatter.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
docformatter.default_interpreter_constraints),
entry_point=docformatter.get_entry_point(),
))
all_source_files_request = Get[SourceFiles](AllSourceFilesRequest(
field_set.sources for field_set in setup_request.request.field_sets))
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest(
(field_set.sources, field_set.origin)
for field_set in setup_request.request.field_sets))
requests: List[Get] = [
requirements_pex_request, specified_source_files_request
]
if setup_request.request.prior_formatter_result is None:
requests.append(all_source_files_request)
requirements_pex, specified_source_files, *rest = cast(
Union[Tuple[Pex, SourceFiles], Tuple[Pex, SourceFiles, SourceFiles]],
await MultiGet(requests),
)
all_source_files_snapshot = (
setup_request.request.prior_formatter_result
if setup_request.request.prior_formatter_result else rest[0].snapshot)
input_digest = await Get[Digest](MergeDigests(
(all_source_files_snapshot.digest, requirements_pex.digest)))
address_references = ", ".join(
sorted(field_set.address.reference()
for field_set in setup_request.request.field_sets))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./docformatter.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
docformatter=docformatter,
check_only=setup_request.check_only,
),
input_digest=input_digest,
output_files=all_source_files_snapshot.files,
description=
(f"Run Docformatter on {pluralize(len(setup_request.request.field_sets), 'target')}: "
f"{address_references}."),
)
return Setup(process, original_digest=all_source_files_snapshot.digest)
async def bandit_lint(
configs: BanditConfigurations,
bandit: Bandit,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if bandit.options.skip:
return LintResult.noop()
# NB: Bandit output depends upon which Python interpreter version it's run with. See
# https://github.com/PyCQA/bandit#under-which-version-of-python-should-i-install-bandit.
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(config.compatibility for config in configs), python_setup=python_setup
)
requirements_pex = await Get[Pex](
PexRequest(
output_filename="bandit.pex",
requirements=PexRequirements(bandit.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=bandit.get_entry_point(),
)
)
config_path: Optional[str] = bandit.options.config
config_snapshot = await Get[Snapshot](
PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--bandit-config`",
)
)
all_source_files = await Get[SourceFiles](
AllSourceFilesRequest(config.sources for config in configs)
)
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest((config.sources, config.origin) for config in configs)
)
merged_input_files = await Get[Digest](
MergeDigests(
(all_source_files.snapshot.digest, requirements_pex.digest, config_snapshot.digest)
),
)
address_references = ", ".join(sorted(config.address.reference() for config in configs))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./bandit.pex",
pex_args=generate_args(specified_source_files=specified_source_files, bandit=bandit),
input_files=merged_input_files,
description=f"Run Bandit on {pluralize(len(configs), 'target')}: {address_references}.",
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result)
async def flake8_lint(
configs: Flake8Configurations,
flake8: Flake8,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if flake8.options.skip:
return LintResult.noop()
# NB: Flake8 output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://flake8.pycqa.org/en/latest/user/invocation.html.
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(config.compatibility for config in configs), python_setup)
requirements_pex = await Get[Pex](PexRequest(
output_filename="flake8.pex",
requirements=PexRequirements(flake8.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=flake8.get_entry_point(),
))
config_path: Optional[str] = flake8.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--flake8-config`",
))
all_source_files = await Get[SourceFiles](AllSourceFilesRequest(
config.sources for config in configs))
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest(
(config.sources, config.origin) for config in configs))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files.snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
address_references = ", ".join(
sorted(config.address.reference() for config in configs))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./flake8.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
flake8=flake8),
input_files=merged_input_files,
description=
f"Run Flake8 on {pluralize(len(configs), 'target')}: {address_references}.",
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result)
def get_specified_source_files(
self,
sources_fields_with_origins: Iterable[Tuple[SourcesField, OriginSpec]],
*,
strip_source_roots: bool = False,
) -> List[str]:
request = SpecifiedSourceFilesRequest(
sources_fields_with_origins,
strip_source_roots=strip_source_roots,
)
result = self.request_single_product(
SourceFiles, Params(request, create_options_bootstrapper()))
return sorted(result.snapshot.files)
async def setup(
request: SetupRequest,
docformatter: Docformatter,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
requirements_pex = await Get[Pex](PexRequest(
output_filename="docformatter.pex",
requirements=PexRequirements(docformatter.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
docformatter.default_interpreter_constraints),
entry_point=docformatter.get_entry_point(),
))
if request.configs.prior_formatter_result is None:
all_source_files = await Get[SourceFiles](AllSourceFilesRequest(
config.sources for config in request.configs))
all_source_files_snapshot = all_source_files.snapshot
else:
all_source_files_snapshot = request.configs.prior_formatter_result
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest(
(config.sources, config.origin) for config in request.configs))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files_snapshot.directory_digest,
requirements_pex.directory_digest,
)), )
address_references = ", ".join(
sorted(config.address.reference() for config in request.configs))
process = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./docformatter.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
docformatter=docformatter,
check_only=request.check_only,
),
input_files=merged_input_files,
output_files=all_source_files_snapshot.files,
description=(
f"Run Docformatter on {pluralize(len(request.configs), 'target')}: "
f"{address_references}."),
)
return Setup(process)
async def setup(
setup_request: SetupRequest,
isort: Isort,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
requirements_pex_request = Get[Pex](PexRequest(
output_filename="isort.pex",
requirements=PexRequirements(isort.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
isort.default_interpreter_constraints),
entry_point=isort.get_entry_point(),
))
config_path: Optional[List[str]] = isort.options.config
config_snapshot_request = Get[Snapshot](PathGlobs(
globs=config_path or (),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin="the option `--isort-config`",
))
all_source_files_request = Get[SourceFiles](AllSourceFilesRequest(
field_set.sources for field_set in setup_request.request.field_sets))
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest(
(field_set.sources, field_set.origin)
for field_set in setup_request.request.field_sets))
requests: List[Get] = [
requirements_pex_request,
config_snapshot_request,
specified_source_files_request,
]
if setup_request.request.prior_formatter_result is None:
requests.append(all_source_files_request)
requirements_pex, config_snapshot, specified_source_files, *rest = cast(
Union[Tuple[Pex, Snapshot, SourceFiles],
Tuple[Pex, Snapshot, SourceFiles, SourceFiles]],
await MultiGet(requests),
)
all_source_files_snapshot = (
setup_request.request.prior_formatter_result
if setup_request.request.prior_formatter_result else rest[0].snapshot)
input_digest = await Get[Digest](MergeDigests(
(all_source_files_snapshot.digest, requirements_pex.digest,
config_snapshot.digest)))
address_references = ", ".join(
sorted(field_set.address.reference()
for field_set in setup_request.request.field_sets))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./isort.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
isort=isort,
check_only=setup_request.check_only,
),
input_digest=input_digest,
output_files=all_source_files_snapshot.files,
description=
(f"Run isort on {pluralize(len(setup_request.request.field_sets), 'target')}: {address_references}."
),
)
return Setup(process, original_digest=all_source_files_snapshot.digest)
async def setup_pytest_for_target(
field_set: PythonTestFieldSet,
pytest: PyTest,
test_options: TestOptions,
python_setup: PythonSetup,
) -> TestTargetSetup:
# TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage
# configuration via #7490.
test_addresses = Addresses((field_set.address, ))
transitive_targets = await Get[TransitiveTargets](Addresses,
test_addresses)
all_targets = transitive_targets.closure
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(tgt[PythonInterpreterCompatibility] for tgt in all_targets
if tgt.has_field(PythonInterpreterCompatibility)),
python_setup,
)
# Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints
# of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for
# CreatePex requests.
pex_request = functools.partial(
PexRequest, interpreter_constraints=interpreter_constraints)
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args_for_pytest = ("--not-zip-safe", )
use_coverage = test_options.values.use_coverage
plugin_file_digest: Optional[Digest] = (await Get[Digest](
InputFilesContent, COVERAGE_PLUGIN_INPUT) if use_coverage else None)
pytest_pex_request = Get[Pex](
PexRequest,
pex_request(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
additional_args=additional_args_for_pytest,
sources=plugin_file_digest,
),
)
requirements_pex_request = Get[Pex](PexFromTargetsRequest(
addresses=test_addresses,
output_filename="requirements.pex",
include_source_files=False,
additional_args=additional_args_for_pytest,
))
test_runner_pex_request = Get[Pex](
PexRequest,
pex_request(
output_filename="test_runner.pex",
entry_point="pytest:main",
interpreter_constraints=interpreter_constraints,
additional_args=(
"--pex-path",
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any pytest transitive requirements will shadow corresponding user
# requirements which will lead to problems when APIs that are used by either
# `pytest:main` or the tests themselves break between the two versions.
":".join((
pytest_pex_request.subject.output_filename,
requirements_pex_request.subject.output_filename,
)),
),
),
)
prepared_sources_request = Get[ImportablePythonSources](
Targets(all_targets))
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest([(field_set.sources, field_set.origin)],
strip_source_roots=True))
requests = (
pytest_pex_request,
requirements_pex_request,
test_runner_pex_request,
prepared_sources_request,
specified_source_files_request,
)
(
coverage_config,
pytest_pex,
requirements_pex,
test_runner_pex,
prepared_sources,
specified_source_files,
) = (await MultiGet(
Get(
CoverageConfig,
CoverageConfigRequest(
Targets((tgt for tgt in all_targets
if tgt.has_field(PythonSources))),
is_test_time=True,
),
),
*requests,
) if use_coverage else
(CoverageConfig(EMPTY_DIGEST), *await MultiGet(*requests)))
digests_to_merge = [
coverage_config.digest,
prepared_sources.snapshot.digest,
requirements_pex.digest,
pytest_pex.digest,
test_runner_pex.digest,
]
input_digest = await Get[Digest](MergeDigests(digests_to_merge))
coverage_args = []
if use_coverage:
coverage_args = [
"--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html
]
for package in field_set.coverage.determine_packages_to_cover(
specified_source_files=specified_source_files):
coverage_args.extend(["--cov", package])
specified_source_file_names = sorted(specified_source_files.snapshot.files)
return TestTargetSetup(
test_runner_pex=test_runner_pex,
args=(*pytest.options.args, *coverage_args,
*specified_source_file_names),
input_digest=input_digest,
timeout_seconds=field_set.timeout.calculate_from_global_options(
pytest),
xml_dir=pytest.options.junit_xml_dir,
junit_family=pytest.options.junit_family,
)
async def setup(
request: SetupRequest,
black: Black,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
requirements_pex = await Get[Pex](
PexRequest(
output_filename="black.pex",
requirements=PexRequirements(black.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
black.default_interpreter_constraints
),
entry_point=black.get_entry_point(),
)
)
config_path: Optional[str] = black.options.config
config_snapshot = await Get[Snapshot](
PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--black-config`",
)
)
if request.configs.prior_formatter_result is None:
all_source_files = await Get[SourceFiles](
AllSourceFilesRequest(config.sources for config in request.configs)
)
all_source_files_snapshot = all_source_files.snapshot
else:
all_source_files_snapshot = request.configs.prior_formatter_result
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest((config.sources, config.origin) for config in request.configs)
)
merged_input_files = await Get[Digest](
MergeDigests(
(all_source_files_snapshot.digest, requirements_pex.digest, config_snapshot.digest)
),
)
address_references = ", ".join(sorted(config.address.reference() for config in request.configs))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./black.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
black=black,
check_only=request.check_only,
),
input_files=merged_input_files,
output_files=all_source_files_snapshot.files,
description=(
f"Run Black on {pluralize(len(request.configs), 'target')}: {address_references}."
),
)
return Setup(process, original_digest=all_source_files_snapshot.digest)
async def pylint_lint(
configs: PylintConfigurations,
pylint: Pylint,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if pylint.options.skip:
return LintResult.noop()
# Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it
# doesn't lint those direct dependencies nor does it care about transitive dependencies.
addresses = []
for config in configs:
addresses.append(config.address)
addresses.extend(config.dependencies.value or ())
targets = await Get[Targets](Addresses(addresses))
chrooted_python_sources = await Get[ImportablePythonSources](Targets,
targets)
# NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting.
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(config.compatibility for config in configs), python_setup)
requirements_pex = await Get[Pex](PexRequest(
output_filename="pylint.pex",
requirements=PexRequirements(pylint.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=pylint.get_entry_point(),
))
config_path: Optional[str] = pylint.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
requirements_pex.directory_digest,
config_snapshot.directory_digest,
chrooted_python_sources.snapshot.directory_digest,
)), )
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest(
((config.sources, config.origin) for config in configs),
strip_source_roots=True))
address_references = ", ".join(
sorted(config.address.reference() for config in configs))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./pylint.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
pylint=pylint),
input_files=merged_input_files,
description=
f"Run Pylint on {pluralize(len(configs), 'target')}: {address_references}.",
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result)
async def setup_pytest_for_target(
config: PythonTestConfiguration,
pytest: PyTest,
test_options: TestOptions,
python_setup: PythonSetup,
) -> TestTargetSetup:
# TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage
# configuration via #7490.
test_addresses = Addresses((config.address, ))
# TODO(John Sirois): PexInterpreterConstraints are gathered in the same way by the
# `create_pex_from_target_closure` rule, factor up.
transitive_targets = await Get[TransitiveTargets](Addresses,
test_addresses)
all_targets = transitive_targets.closure
# TODO: factor this up? It's mostly duplicated with pex_from_targets.py.
python_targets = []
resource_targets = []
for tgt in all_targets:
if tgt.has_field(PythonSources):
python_targets.append(tgt)
# NB: PythonRequirementsFileSources is a subclass of FilesSources. We filter it out so that
# requirements.txt is not included in the PEX and so that irrelevant changes to it (e.g.
# whitespace changes) do not invalidate the PEX.
if tgt.has_field(ResourcesSources) or (
tgt.has_field(FilesSources)
and not tgt.has_field(PythonRequirementsFileSources)):
resource_targets.append(tgt)
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(tgt.get(PythonInterpreterCompatibility) for tgt in python_targets),
python_setup)
# Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints
# of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for
# CreatePex requests.
pex_request = functools.partial(
PexRequest, interpreter_constraints=interpreter_constraints)
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args_for_pytest = ("--not-zip-safe", )
run_coverage = test_options.values.run_coverage
plugin_file_digest: Optional[Digest] = (await Get[Digest](
InputFilesContent, COVERAGE_PLUGIN_INPUT) if run_coverage else None)
pytest_pex_request = pex_request(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
additional_args=additional_args_for_pytest,
sources=plugin_file_digest,
)
requirements_pex_request = PexFromTargetsRequest(
addresses=test_addresses,
output_filename="requirements.pex",
include_source_files=False,
additional_args=additional_args_for_pytest,
)
test_runner_pex_request = pex_request(
output_filename="test_runner.pex",
entry_point="pytest:main",
interpreter_constraints=interpreter_constraints,
additional_args=(
"--pex-path",
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any pytest transitive requirements will shadow corresponding user
# requirements which will lead to problems when APIs that are used by either
# `pytest:main` or the tests themselves break between the two versions.
":".join((pytest_pex_request.output_filename,
requirements_pex_request.output_filename)),
),
)
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
specified_source_files_request = SpecifiedSourceFilesRequest(
[(config.sources, config.origin)], strip_source_roots=True)
# TODO(John Sirois): Support exploiting concurrency better:
# https://github.com/pantsbuild/pants/issues/9294
# Some awkward code follows in order to execute 5-6 items concurrently given the current state
# of MultiGet typing / API. Improve this since we should encourage full concurrency in general.
requests: List[Get[Any]] = [
Get[Pex](PexRequest, pytest_pex_request),
Get[Pex](PexFromTargetsRequest, requirements_pex_request),
Get[Pex](PexRequest, test_runner_pex_request),
Get[ImportablePythonSources](Targets(python_targets +
resource_targets)),
Get[SourceFiles](SpecifiedSourceFilesRequest,
specified_source_files_request),
]
if run_coverage:
requests.append(
Get[CoverageConfig](CoverageConfigRequest(Targets(python_targets),
is_test_time=True)), )
(
pytest_pex,
requirements_pex,
test_runner_pex,
prepared_sources,
specified_source_files,
*rest,
) = cast(
Union[Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles],
Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles,
CoverageConfig], ],
await MultiGet(requests),
)
directories_to_merge = [
prepared_sources.snapshot.directory_digest,
requirements_pex.directory_digest,
pytest_pex.directory_digest,
test_runner_pex.directory_digest,
]
if run_coverage:
coverage_config = rest[0]
directories_to_merge.append(coverage_config.digest)
merged_input_files = await Get[Digest](
DirectoriesToMerge(directories=tuple(directories_to_merge)))
coverage_args = []
if run_coverage:
coverage_args = [
"--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html
]
for package in config.coverage.determine_packages_to_cover(
specified_source_files=specified_source_files):
coverage_args.extend(["--cov", package])
specified_source_file_names = sorted(specified_source_files.snapshot.files)
return TestTargetSetup(
test_runner_pex=test_runner_pex,
args=(*pytest.options.args, *coverage_args,
*specified_source_file_names),
input_files_digest=merged_input_files,
timeout_seconds=config.timeout.calculate_from_global_options(pytest),
)
async def pylint_lint_partition(
partition: PylintPartition,
pylint: Pylint,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
# We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies.
# Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`.
plugin_requirements = PexRequirements.create_from_requirement_fields(
plugin_tgt[PythonRequirementsField]
for plugin_tgt in partition.plugin_targets
if plugin_tgt.has_field(PythonRequirementsField))
target_requirements = PexRequirements.create_from_requirement_fields(
tgt[PythonRequirementsField]
for tgt in partition.targets_with_dependencies
if tgt.has_field(PythonRequirementsField))
pylint_pex_request = Get[Pex](PexRequest(
output_filename="pylint.pex",
requirements=PexRequirements(
[*pylint.get_requirement_specs(), *plugin_requirements]),
interpreter_constraints=partition.interpreter_constraints,
entry_point=pylint.get_entry_point(),
))
requirements_pex_request = Get[Pex](PexRequest(
output_filename="requirements.pex",
requirements=target_requirements,
interpreter_constraints=partition.interpreter_constraints,
))
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any Pylint transitive requirements will shadow corresponding user
# requirements, which could lead to problems.
pylint_runner_pex_args = [
"--pex-path", ":".join(["pylint.pex", "requirements.pex"])
]
if pylint.source_plugins:
# NB: See below for why we set PYTHONPATH to load source plugins. This setting is necessary
# for PEX to pick up the PYTHONPATH value.
pylint_runner_pex_args.append("--inherit-path=fallback")
pylint_runner_pex_request = Get[Pex](PexRequest(
output_filename="pylint_runner.pex",
entry_point=pylint.get_entry_point(),
interpreter_constraints=partition.interpreter_constraints,
additional_args=pylint_runner_pex_args,
))
config_snapshot_request = Get[Snapshot](PathGlobs(
globs=[pylint.config] if pylint.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
))
prepare_plugin_sources_request = Get[ImportablePythonSources](
Targets, partition.plugin_targets)
prepare_python_sources_request = Get[ImportablePythonSources](
Targets, partition.targets_with_dependencies)
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest(
((field_set.sources, field_set.origin)
for field_set in partition.field_sets),
strip_source_roots=True,
))
(
pylint_pex,
requirements_pex,
pylint_runner_pex,
config_snapshot,
prepared_plugin_sources,
prepared_python_sources,
specified_source_files,
) = await MultiGet(
pylint_pex_request,
requirements_pex_request,
pylint_runner_pex_request,
config_snapshot_request,
prepare_plugin_sources_request,
prepare_python_sources_request,
specified_source_files_request,
)
prefixed_plugin_sources = (await Get[Digest](AddPrefix(
prepared_plugin_sources.snapshot.digest, "__plugins"))
if pylint.source_plugins else EMPTY_DIGEST)
input_digest = await Get[Digest](MergeDigests((
pylint_pex.digest,
requirements_pex.digest,
pylint_runner_pex.digest,
config_snapshot.digest,
prefixed_plugin_sources,
prepared_python_sources.snapshot.digest,
)), )
address_references = ", ".join(
sorted(field_set.address.reference()
for field_set in partition.field_sets))
process = pylint_runner_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./pylint_runner.pex",
# NB: Pylint source plugins must be explicitly loaded via PYTHONPATH. The value must
# point to the plugin's directory, rather than to a parent's directory, because
# `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but
# not `path.to.plugin`. (This means users must have specified the parent directory as a
# source root.)
env={"PYTHONPATH": "./__plugins"} if pylint.source_plugins else None,
pex_args=generate_args(specified_source_files=specified_source_files,
pylint=pylint),
input_digest=input_digest,
description=
(f"Run Pylint on {pluralize(len(partition.field_sets), 'target')}: {address_references}."
),
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result,
linter_name="Pylint")
async def pylint_lint(
field_sets: PylintFieldSets,
pylint: Pylint,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if pylint.skip:
return LintResult.noop()
# Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it
# doesn't lint those direct dependencies nor does it care about transitive dependencies.
addresses_with_dependencies = []
for field_set in field_sets:
addresses_with_dependencies.append(field_set.address)
addresses_with_dependencies.extend(field_set.dependencies.value or ())
targets = await Get[Targets](Addresses(addresses_with_dependencies))
# NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting.
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(field_set.compatibility for field_set in field_sets), python_setup)
# We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies.
# Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`.
pylint_pex_request = Get[Pex](PexRequest(
output_filename="pylint.pex",
requirements=PexRequirements(pylint.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=pylint.get_entry_point(),
))
requirements_pex_request = Get[Pex](PexRequest(
output_filename="requirements.pex",
requirements=PexRequirements.create_from_requirement_fields(
tgt[PythonRequirementsField] for tgt in targets
if tgt.has_field(PythonRequirementsField)),
interpreter_constraints=interpreter_constraints,
))
pylint_runner_pex_request = Get[Pex](
PexRequest(
output_filename="pylint_runner.pex",
entry_point=pylint.get_entry_point(),
interpreter_constraints=interpreter_constraints,
additional_args=(
"--pex-path",
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any Pylint transitive requirements will shadow corresponding user
# requirements which could lead to problems.
":".join(["pylint.pex", "requirements.pex"]),
),
))
config_snapshot_request = Get[Snapshot](PathGlobs(
globs=[pylint.config] if pylint.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
))
prepare_python_sources_request = Get[ImportablePythonSources](Targets,
targets)
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest(
((field_set.sources, field_set.origin)
for field_set in field_sets),
strip_source_roots=True,
))
(
pylint_pex,
requirements_pex,
pylint_runner_pex,
config_snapshot,
prepared_python_sources,
specified_source_files,
) = cast(
Tuple[Pex, Pex, Pex, Snapshot, ImportablePythonSources, SourceFiles],
await MultiGet([
pylint_pex_request,
requirements_pex_request,
pylint_runner_pex_request,
config_snapshot_request,
prepare_python_sources_request,
specified_source_files_request,
]),
)
input_digest = await Get[Digest](MergeDigests((
pylint_pex.digest,
requirements_pex.digest,
pylint_runner_pex.digest,
config_snapshot.digest,
prepared_python_sources.snapshot.digest,
)), )
address_references = ", ".join(
sorted(field_set.address.reference() for field_set in field_sets))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./pylint_runner.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
pylint=pylint),
input_digest=input_digest,
description=
f"Run Pylint on {pluralize(len(field_sets), 'target')}: {address_references}.",
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result,
linter_name="Pylint")
