async def setup(
request: SetupRequest,
isort: Isort,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
adaptors_with_origins = request.formatter.adaptors_with_origins
requirements_pex = await Get[Pex](PexRequest(
output_filename="isort.pex",
requirements=PexRequirements(isort.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
isort.default_interpreter_constraints),
entry_point=isort.get_entry_point(),
))
config_path: Optional[List[str]] = isort.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=config_path or (),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin="the option `--isort-config`",
))
if request.formatter.prior_formatter_result is None:
all_source_files = await Get[SourceFiles](LegacyAllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
all_source_files_snapshot = all_source_files.snapshot
else:
all_source_files_snapshot = request.formatter.prior_formatter_result
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files_snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
process_request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./isort.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
isort=isort,
check_only=request.check_only,
),
input_files=merged_input_files,
output_files=all_source_files_snapshot.files,
description=f"Run isort for {address_references}",
)
return Setup(process_request)
def get_specified_source_files(
self,
adaptors_with_origins: Iterable[TargetAdaptorWithOrigin],
*,
strip_source_roots: bool = False,
) -> List[str]:
request = LegacySpecifiedSourceFilesRequest(
adaptors_with_origins, strip_source_roots=strip_source_roots,
)
result = self.request_single_product(
SourceFiles, Params(request, create_options_bootstrapper())
)
return sorted(result.snapshot.files)
async def setup(
request: SetupRequest,
docformatter: Docformatter,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
adaptors_with_origins = request.formatter.adaptors_with_origins
requirements_pex = await Get[Pex](PexRequest(
output_filename="docformatter.pex",
requirements=PexRequirements(docformatter.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
docformatter.default_interpreter_constraints),
entry_point=docformatter.get_entry_point(),
))
if request.formatter.prior_formatter_result is None:
all_source_files = await Get[SourceFiles](LegacyAllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
all_source_files_snapshot = all_source_files.snapshot
else:
all_source_files_snapshot = request.formatter.prior_formatter_result
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files_snapshot.directory_digest,
requirements_pex.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
process_request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./docformatter.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
docformatter=docformatter,
check_only=request.check_only,
),
input_files=merged_input_files,
output_files=all_source_files_snapshot.files,
description=f"Run docformatter for {address_references}",
)
return Setup(process_request)
async def lint(
linter: Flake8Linter,
flake8: Flake8,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if flake8.options.skip:
return LintResult.noop()
adaptors_with_origins = linter.adaptors_with_origins
# NB: Flake8 output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://flake8.pycqa.org/en/latest/user/invocation.html.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
(adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins),
python_setup=python_setup,
)
requirements_pex = await Get[Pex](PexRequest(
output_filename="flake8.pex",
requirements=PexRequirements(flake8.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=flake8.get_entry_point(),
))
config_path: Optional[str] = flake8.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--flake8-config`",
))
all_source_files = await Get[SourceFiles](LegacyAllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files.snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./flake8.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
flake8=flake8),
input_files=merged_input_files,
description=f"Run Flake8 for {address_references}",
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return LintResult.from_fallible_execute_process_result(result)
async def lint(
linter: PylintLinter,
pylint: Pylint,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if pylint.options.skip:
return LintResult.noop()
adaptors_with_origins = linter.adaptors_with_origins
# Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it
# doesn't lint those direct dependencies nor does it care about transitive dependencies.
hydrated_targets = [
HydratedTarget(adaptor_with_origin.adaptor)
for adaptor_with_origin in adaptors_with_origins
]
dependencies = await MultiGet(
Get[HydratedTarget](Address, dependency)
for dependency in itertools.chain.from_iterable(
ht.adaptor.dependencies for ht in hydrated_targets))
chrooted_python_sources = await Get[ImportablePythonSources](
HydratedTargets([*hydrated_targets, *dependencies]))
# NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
(adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins),
python_setup=python_setup,
)
requirements_pex = await Get[Pex](PexRequest(
output_filename="pylint.pex",
requirements=PexRequirements(pylint.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=pylint.get_entry_point(),
))
config_path: Optional[str] = pylint.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
requirements_pex.directory_digest,
config_snapshot.directory_digest,
chrooted_python_sources.snapshot.directory_digest,
)), )
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins,
strip_source_roots=True))
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./pylint.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
pylint=pylint),
input_files=merged_input_files,
description=f"Run Pylint for {address_references}",
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return LintResult.from_fallible_execute_process_result(result)
async def setup_pytest_for_target(
pytest_runner: PytestRunner,
pytest: PyTest,
test_options: TestOptions,
python_setup: PythonSetup,
) -> TestTargetSetup:
# TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage
# configuration via #7490.
adaptor_with_origin = pytest_runner.adaptor_with_origin
adaptor = adaptor_with_origin.adaptor
test_addresses = Addresses((adaptor.address,))
# TODO(John Sirois): PexInterpreterConstraints are gathered in the same way by the
# `create_pex_from_target_closure` rule, factor up.
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](Addresses, test_addresses)
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=all_target_adaptors, python_setup=python_setup
)
# Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints
# of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for
# CreatePex requests.
pex_request = functools.partial(PexRequest, interpreter_constraints=interpreter_constraints)
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args_for_pytest = ("--not-zip-safe",)
run_coverage = test_options.values.run_coverage
plugin_file_digest: Optional[Digest] = (
await Get[Digest](InputFilesContent, get_coverage_plugin_input()) if run_coverage else None
)
pytest_pex_request = pex_request(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
additional_args=additional_args_for_pytest,
input_files_digest=plugin_file_digest,
)
requirements_pex_request = LegacyPexFromTargetsRequest(
addresses=test_addresses,
output_filename="requirements.pex",
include_source_files=False,
additional_args=additional_args_for_pytest,
)
test_runner_pex_request = pex_request(
output_filename="test_runner.pex",
entry_point="pytest:main",
interpreter_constraints=interpreter_constraints,
additional_args=(
"--pex-path",
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any pytest transitive requirements will shadow corresponding user
# requirements which will lead to problems when APIs that are used by either
# `pytest:main` or the tests themselves break between the two versions.
":".join(
(pytest_pex_request.output_filename, requirements_pex_request.output_filename)
),
),
)
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
specified_source_files_request = LegacySpecifiedSourceFilesRequest(
[adaptor_with_origin], strip_source_roots=True
)
# TODO: Replace this with appropriate target API logic.
python_targets = [t for t in all_targets if isinstance(t.adaptor, PythonTargetAdaptor)]
resource_targets = [
t for t in all_targets if isinstance(t.adaptor, (FilesAdaptor, ResourcesAdaptor))
]
# TODO(John Sirois): Support exploiting concurrency better:
# https://github.com/pantsbuild/pants/issues/9294
# Some awkward code follows in order to execute 5-6 items concurrently given the current state
# of MultiGet typing / API. Improve this since we should encourage full concurrency in general.
requests: List[Get[Any]] = [
Get[Pex](PexRequest, pytest_pex_request),
Get[Pex](LegacyPexFromTargetsRequest, requirements_pex_request),
Get[Pex](PexRequest, test_runner_pex_request),
Get[ImportablePythonSources](HydratedTargets(python_targets + resource_targets)),
Get[SourceFiles](LegacySpecifiedSourceFilesRequest, specified_source_files_request),
]
if run_coverage:
requests.append(
Get[Coveragerc](CoveragercRequest(HydratedTargets(python_targets), test_time=True)),
)
(
pytest_pex,
requirements_pex,
test_runner_pex,
chrooted_sources,
specified_source_files,
*rest,
) = cast(
Union[
Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles],
Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles, Coveragerc],
],
await MultiGet(requests),
)
directories_to_merge = [
chrooted_sources.snapshot.directory_digest,
requirements_pex.directory_digest,
pytest_pex.directory_digest,
test_runner_pex.directory_digest,
]
if run_coverage:
coveragerc = rest[0]
directories_to_merge.append(coveragerc.digest)
merged_input_files = await Get[Digest](
DirectoriesToMerge(directories=tuple(directories_to_merge))
)
coverage_args = []
if run_coverage:
coverage_args = [
"--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html
]
# TODO: replace this with proper usage of the Target API.
coverage_field = PythonCoverage(getattr(adaptor, "coverage", None), address=adaptor.address)
for package in coverage_field.determine_packages_to_cover(
specified_source_files=specified_source_files
):
coverage_args.extend(["--cov", package])
# TODO: replace this with proper usage of the Target API.
timeout_field = PythonTestsTimeout(getattr(adaptor, "timeout", None), address=adaptor.address)
specified_source_file_names = sorted(specified_source_files.snapshot.files)
return TestTargetSetup(
test_runner_pex=test_runner_pex,
args=(*pytest.options.args, *coverage_args, *specified_source_file_names),
input_files_digest=merged_input_files,
timeout_seconds=timeout_field.calculate_from_global_options(pytest),
)
async def bandit_lint(
linter: BanditLinter,
bandit: Bandit,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if bandit.options.skip:
return LintResult.noop()
adaptors_with_origins = linter.adaptors_with_origins
# NB: Bandit output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# https://github.com/PyCQA/bandit#under-which-version-of-python-should-i-install-bandit.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
(adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins),
python_setup=python_setup,
)
requirements_pex = await Get[Pex](PexRequest(
output_filename="bandit.pex",
requirements=PexRequirements(bandit.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=bandit.get_entry_point(),
))
config_path: Optional[str] = bandit.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--bandit-config`",
))
all_source_files = await Get[SourceFiles](LegacyAllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files.snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./bandit.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
bandit=bandit),
input_files=merged_input_files,
description=f"Run Bandit for {address_references}",
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return LintResult.from_fallible_execute_process_result(result)