async def flake8_lint_partition(partition: Flake8Partition, flake8: Flake8) -> LintResult:
flake8_pex_get = Get(
VenvPex,
PexRequest(
output_filename="flake8.pex",
internal_only=True,
requirements=flake8.pex_requirements(),
interpreter_constraints=partition.interpreter_constraints,
main=flake8.main,
),
)
config_files_get = Get(ConfigFiles, ConfigFilesRequest, flake8.config_request)
source_files_get = Get(
SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)
)
# Ensure that the empty report dir exists.
report_directory_digest_get = Get(Digest, CreateDigest([Directory(REPORT_DIR)]))
flake8_pex, config_files, report_directory, source_files = await MultiGet(
flake8_pex_get, config_files_get, report_directory_digest_get, source_files_get
)
input_digest = await Get(
Digest,
MergeDigests(
(source_files.snapshot.digest, config_files.snapshot.digest, report_directory)
),
)
result = await Get(
FallibleProcessResult,
VenvPexProcess(
flake8_pex,
argv=generate_argv(source_files, flake8),
input_digest=input_digest,
output_directories=(REPORT_DIR,),
description=f"Run Flake8 on {pluralize(len(partition.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR))
return LintResult.from_fallible_process_result(
result,
partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def google_java_format_lint(
request: GoogleJavaFormatRequest,
tool: GoogleJavaFormatSubsystem) -> LintResults:
if tool.skip:
return LintResults([], linter_name=request.name)
setup = await Get(Setup, SetupRequest(request, check_only=True))
result = await Get(FallibleProcessResult, JvmProcess, setup.process)
lint_result = LintResult.from_fallible_process_result(result)
if lint_result.exit_code == 0 and lint_result.stdout.strip() != "":
# Note: The formetter returns success even if it would have reformatted the files.
# When this occurs, convert the LintResult into a failure.
lint_result = dataclasses.replace(
lint_result,
exit_code=1,
stdout=
f"The following Java files require formatting:\n{lint_result.stdout}\n",
)
return LintResults([lint_result], linter_name=request.name)
async def run_hadolint(request: HadolintRequest,
hadolint: Hadolint) -> LintResults:
if hadolint.skip:
return LintResults([], linter_name="Hadolint")
downloaded_hadolint, sources, config_files = await MultiGet(
Get(DownloadedExternalTool, ExternalToolRequest,
hadolint.get_request(Platform.current)),
Get(
SourceFiles,
SourceFilesRequest(
[field_set.sources for field_set in request.field_sets],
for_sources_types=(DockerImageSources, ),
enable_codegen=True,
),
),
Get(ConfigFiles, ConfigFilesRequest, hadolint.config_request()),
)
input_digest = await Get(
Digest,
MergeDigests((
sources.snapshot.digest,
downloaded_hadolint.digest,
config_files.snapshot.digest,
)),
)
process_result = await Get(
FallibleProcessResult,
Process(
argv=[downloaded_hadolint.exe, *generate_argv(sources, hadolint)],
input_digest=input_digest,
description=
f"Run `hadolint` on {pluralize(len(sources.files), 'Dockerfile')}.",
level=LogLevel.DEBUG,
),
)
return LintResults(
[LintResult.from_fallible_process_result(process_result)],
linter_name="hadolint")
async def run_shellcheck(request: ShellcheckRequest,
shellcheck: Shellcheck) -> LintResults:
if shellcheck.skip:
return LintResults([], linter_name="Shellcheck")
# Shellcheck looks at direct dependencies to make sure that every symbol is defined, so we must
# include those in the run.
all_dependencies = await MultiGet(
Get(Targets, DependenciesRequest(field_set.dependencies))
for field_set in request.field_sets)
direct_sources_get = Get(
SourceFiles,
SourceFilesRequest(
(field_set.sources for field_set in request.field_sets),
for_sources_types=(ShellSourcesField, ),
enable_codegen=True,
),
)
dependency_sources_get = Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for dependencies in all_dependencies
for tgt in dependencies),
for_sources_types=(ShellSourcesField, ),
enable_codegen=True,
),
)
download_shellcheck_get = Get(DownloadedExternalTool, ExternalToolRequest,
shellcheck.get_request(Platform.current))
direct_sources, dependency_sources, downloaded_shellcheck = await MultiGet(
direct_sources_get, dependency_sources_get, download_shellcheck_get)
config_files = await Get(
ConfigFiles, ConfigFilesRequest,
shellcheck.config_request(direct_sources.snapshot.dirs))
input_digest = await Get(
Digest,
MergeDigests((
direct_sources.snapshot.digest,
dependency_sources.snapshot.digest,
downloaded_shellcheck.digest,
config_files.snapshot.digest,
)),
)
process_result = await Get(
FallibleProcessResult,
Process(
argv=[
downloaded_shellcheck.exe, *shellcheck.args,
*direct_sources.snapshot.files
],
input_digest=input_digest,
description=
f"Run Shellcheck on {pluralize(len(request.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
result = LintResult.from_fallible_process_result(process_result)
return LintResults([result], linter_name="Shellcheck")
async def bandit_lint(
field_sets: BanditFieldSets,
bandit: Bandit,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if bandit.options.skip:
return LintResult.noop()
# NB: Bandit output depends upon which Python interpreter version it's run with. See
# https://github.com/PyCQA/bandit#under-which-version-of-python-should-i-install-bandit.
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(field_set.compatibility for field_set in field_sets),
python_setup=python_setup)
requirements_pex_request = Get[Pex](PexRequest(
output_filename="bandit.pex",
requirements=PexRequirements(bandit.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=bandit.get_entry_point(),
))
config_path: Optional[str] = bandit.options.config
config_snapshot_request = Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--bandit-config`",
))
all_source_files_request = Get[SourceFiles](AllSourceFilesRequest(
field_set.sources for field_set in field_sets))
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest(
(field_set.sources, field_set.origin) for field_set in field_sets))
requirements_pex, config_snapshot, all_source_files, specified_source_files = cast(
Tuple[Pex, Snapshot, SourceFiles, SourceFiles],
await MultiGet([
requirements_pex_request,
config_snapshot_request,
all_source_files_request,
specified_source_files_request,
]),
)
input_digest = await Get[Digest](MergeDigests(
(all_source_files.snapshot.digest, requirements_pex.digest,
config_snapshot.digest)))
address_references = ", ".join(
sorted(field_set.address.reference() for field_set in field_sets))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./bandit.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
bandit=bandit),
input_digest=input_digest,
description=
f"Run Bandit on {pluralize(len(field_sets), 'target')}: {address_references}.",
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result,
linter_name="Bandit")
async def pylint_lint(
configs: PylintConfigurations,
pylint: Pylint,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if pylint.options.skip:
return LintResult.noop()
# Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it
# doesn't lint those direct dependencies nor does it care about transitive dependencies.
addresses = []
for config in configs:
addresses.append(config.address)
addresses.extend(config.dependencies.value or ())
targets = await Get[Targets](Addresses(addresses))
chrooted_python_sources = await Get[ImportablePythonSources](Targets,
targets)
# NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting.
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(config.compatibility for config in configs), python_setup)
requirements_pex = await Get[Pex](PexRequest(
output_filename="pylint.pex",
requirements=PexRequirements(pylint.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=pylint.get_entry_point(),
))
config_path: Optional[str] = pylint.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
requirements_pex.directory_digest,
config_snapshot.directory_digest,
chrooted_python_sources.snapshot.directory_digest,
)), )
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest(
((config.sources, config.origin) for config in configs),
strip_source_roots=True))
address_references = ", ".join(
sorted(config.address.reference() for config in configs))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./pylint.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
pylint=pylint),
input_files=merged_input_files,
description=
f"Run Pylint on {pluralize(len(configs), 'target')}: {address_references}.",
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result)
async def pylint_lint_partition(
partition: PylintPartition, pylint: Pylint,
first_party_plugins: PylintFirstPartyPlugins) -> LintResult:
requirements_pex_get = Get(
Pex,
RequirementsPexRequest(
(t.address for t in partition.root_targets),
# NB: These constraints must be identical to the other PEXes. Otherwise, we risk using
# a different version for the requirements than the other two PEXes, which can result
# in a PEX runtime error about missing dependencies.
hardcoded_interpreter_constraints=partition.
interpreter_constraints,
internal_only=True,
),
)
pylint_pex_get = Get(
Pex,
PexRequest,
pylint.to_pex_request(
interpreter_constraints=partition.interpreter_constraints,
extra_requirements=first_party_plugins.requirement_strings,
),
)
prepare_python_sources_get = Get(
PythonSourceFiles, PythonSourceFilesRequest(partition.closure))
field_set_sources_get = Get(
SourceFiles,
SourceFilesRequest(t[PythonSourceField]
for t in partition.root_targets))
# Ensure that the empty report dir exists.
report_directory_digest_get = Get(Digest,
CreateDigest([Directory(REPORT_DIR)]))
(
pylint_pex,
requirements_pex,
prepared_python_sources,
field_set_sources,
report_directory,
) = await MultiGet(
pylint_pex_get,
requirements_pex_get,
prepare_python_sources_get,
field_set_sources_get,
report_directory_digest_get,
)
pylint_runner_pex, config_files = await MultiGet(
Get(
VenvPex,
VenvPexRequest(
PexRequest(
output_filename="pylint_runner.pex",
interpreter_constraints=partition.interpreter_constraints,
main=pylint.main,
internal_only=True,
pex_path=[pylint_pex, requirements_pex],
),
# TODO(John Sirois): Remove this (change to the default of symlinks) when we can
# upgrade to a version of Pylint with https://github.com/PyCQA/pylint/issues/1470
# resolved.
site_packages_copies=True,
),
),
Get(ConfigFiles, ConfigFilesRequest,
pylint.config_request(field_set_sources.snapshot.dirs)),
)
pythonpath = list(prepared_python_sources.source_roots)
if first_party_plugins:
pythonpath.append(first_party_plugins.PREFIX)
input_digest = await Get(
Digest,
MergeDigests((
config_files.snapshot.digest,
first_party_plugins.sources_digest,
prepared_python_sources.source_files.snapshot.digest,
report_directory,
)),
)
result = await Get(
FallibleProcessResult,
VenvPexProcess(
pylint_runner_pex,
argv=generate_argv(field_set_sources, pylint),
input_digest=input_digest,
output_directories=(REPORT_DIR, ),
extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},
concurrency_available=len(partition.root_targets),
description=
f"Run Pylint on {pluralize(len(partition.root_targets), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR))
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
async def pylint_lint(
field_sets: PylintFieldSets,
pylint: Pylint,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if pylint.skip:
return LintResult.noop()
# Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it
# doesn't lint those direct dependencies nor does it care about transitive dependencies.
addresses_with_dependencies = []
for field_set in field_sets:
addresses_with_dependencies.append(field_set.address)
addresses_with_dependencies.extend(field_set.dependencies.value or ())
targets = await Get[Targets](Addresses(addresses_with_dependencies))
# NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting.
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(field_set.compatibility for field_set in field_sets), python_setup)
# We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies.
# Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`.
pylint_pex_request = Get[Pex](PexRequest(
output_filename="pylint.pex",
requirements=PexRequirements(pylint.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=pylint.get_entry_point(),
))
requirements_pex_request = Get[Pex](PexRequest(
output_filename="requirements.pex",
requirements=PexRequirements.create_from_requirement_fields(
tgt[PythonRequirementsField] for tgt in targets
if tgt.has_field(PythonRequirementsField)),
interpreter_constraints=interpreter_constraints,
))
pylint_runner_pex_request = Get[Pex](
PexRequest(
output_filename="pylint_runner.pex",
entry_point=pylint.get_entry_point(),
interpreter_constraints=interpreter_constraints,
additional_args=(
"--pex-path",
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any Pylint transitive requirements will shadow corresponding user
# requirements which could lead to problems.
":".join(["pylint.pex", "requirements.pex"]),
),
))
config_snapshot_request = Get[Snapshot](PathGlobs(
globs=[pylint.config] if pylint.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
))
prepare_python_sources_request = Get[ImportablePythonSources](Targets,
targets)
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest(
((field_set.sources, field_set.origin)
for field_set in field_sets),
strip_source_roots=True,
))
(
pylint_pex,
requirements_pex,
pylint_runner_pex,
config_snapshot,
prepared_python_sources,
specified_source_files,
) = cast(
Tuple[Pex, Pex, Pex, Snapshot, ImportablePythonSources, SourceFiles],
await MultiGet([
pylint_pex_request,
requirements_pex_request,
pylint_runner_pex_request,
config_snapshot_request,
prepare_python_sources_request,
specified_source_files_request,
]),
)
input_digest = await Get[Digest](MergeDigests((
pylint_pex.digest,
requirements_pex.digest,
pylint_runner_pex.digest,
config_snapshot.digest,
prepared_python_sources.snapshot.digest,
)), )
address_references = ", ".join(
sorted(field_set.address.reference() for field_set in field_sets))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./pylint_runner.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
pylint=pylint),
input_digest=input_digest,
description=
f"Run Pylint on {pluralize(len(field_sets), 'target')}: {address_references}.",
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result,
linter_name="Pylint")
async def flake8_lint_partition(
partition: Flake8Partition, flake8: Flake8, lint_subsystem: LintSubsystem
) -> LintResult:
requirements_pex_request = Get(
Pex,
PexRequest(
output_filename="flake8.pex",
internal_only=True,
requirements=PexRequirements(flake8.all_requirements),
interpreter_constraints=(
partition.interpreter_constraints
or PexInterpreterConstraints(flake8.interpreter_constraints)
),
entry_point=flake8.entry_point,
),
)
config_digest_request = Get(
Digest,
PathGlobs(
globs=[flake8.config] if flake8.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--flake8-config`",
),
)
source_files_request = Get(
SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)
)
requirements_pex, config_digest, source_files = await MultiGet(
requirements_pex_request, config_digest_request, source_files_request
)
input_digest = await Get(
Digest,
MergeDigests((source_files.snapshot.digest, requirements_pex.digest, config_digest)),
)
report_file_name = "flake8_report.txt" if lint_subsystem.reports_dir else None
result = await Get(
FallibleProcessResult,
PexProcess(
requirements_pex,
argv=generate_args(
source_files=source_files, flake8=flake8, report_file_name=report_file_name
),
input_digest=input_digest,
output_files=(report_file_name,) if report_file_name else None,
description=f"Run Flake8 on {pluralize(len(partition.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
report = None
if report_file_name:
report_digest = await Get(
Digest,
DigestSubset(
result.output_digest,
PathGlobs(
[report_file_name],
glob_match_error_behavior=GlobMatchErrorBehavior.warn,
description_of_origin="Flake8 report file",
),
),
)
report = LintReport(report_file_name, report_digest)
return LintResult.from_fallible_process_result(
result, partition_description=str(sorted(partition.interpreter_constraints)), report=report
)
async def pylint_lint_partition(partition: PylintPartition,
pylint: Pylint) -> LintResult:
# We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies.
# Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`.
plugin_requirements = PexRequirements.create_from_requirement_fields(
plugin_tgt[PythonRequirementsField]
for plugin_tgt in partition.plugin_targets
if plugin_tgt.has_field(PythonRequirementsField))
target_requirements = PexRequirements.create_from_requirement_fields(
tgt[PythonRequirementsField]
for tgt in partition.targets_with_dependencies
if tgt.has_field(PythonRequirementsField))
pylint_pex_request = Get(
Pex,
PexRequest(
output_filename="pylint.pex",
requirements=PexRequirements(
[*pylint.all_requirements, *plugin_requirements]),
interpreter_constraints=partition.interpreter_constraints,
),
)
requirements_pex_request = Get(
Pex,
PexRequest(
output_filename="requirements.pex",
requirements=target_requirements,
interpreter_constraints=partition.interpreter_constraints,
),
)
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any Pylint transitive requirements will shadow corresponding user
# requirements, which could lead to problems.
pylint_runner_pex_args = [
"--pex-path", ":".join(["pylint.pex", "requirements.pex"])
]
pylint_runner_pex_request = Get(
Pex,
PexRequest(
output_filename="pylint_runner.pex",
entry_point=pylint.entry_point,
interpreter_constraints=partition.interpreter_constraints,
additional_args=pylint_runner_pex_args,
),
)
config_digest_request = Get(
Digest,
PathGlobs(
globs=[pylint.config] if pylint.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
),
)
prepare_plugin_sources_request = Get(
StrippedPythonSourceFiles,
PythonSourceFilesRequest(partition.plugin_targets),
)
prepare_python_sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(partition.targets_with_dependencies),
)
field_set_sources_request = Get(
SourceFiles,
SourceFilesRequest(field_set.sources
for field_set in partition.field_sets),
)
(
pylint_pex,
requirements_pex,
pylint_runner_pex,
config_digest,
prepared_plugin_sources,
prepared_python_sources,
field_set_sources,
) = await MultiGet(
pylint_pex_request,
requirements_pex_request,
pylint_runner_pex_request,
config_digest_request,
prepare_plugin_sources_request,
prepare_python_sources_request,
field_set_sources_request,
)
prefixed_plugin_sources = (await Get(
Digest,
AddPrefix(
prepared_plugin_sources.stripped_source_files.snapshot.digest,
"__plugins"),
) if pylint.source_plugins else EMPTY_DIGEST)
pythonpath = list(prepared_python_sources.source_roots)
if pylint.source_plugins:
# NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must
# point to the plugin's directory, rather than to a parent's directory, because
# `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but
# not `path.to.plugin`. (This means users must have specified the parent directory as a
# source root.)
pythonpath.append("__plugins")
input_digest = await Get(
Digest,
MergeDigests((
pylint_pex.digest,
requirements_pex.digest,
pylint_runner_pex.digest,
config_digest,
prefixed_plugin_sources,
prepared_python_sources.source_files.snapshot.digest,
)),
)
address_references = ", ".join(
sorted(field_set.address.spec for field_set in partition.field_sets))
result = await Get(
FallibleProcessResult,
PexProcess(
pylint_runner_pex,
argv=generate_args(source_files=field_set_sources, pylint=pylint),
input_digest=input_digest,
extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},
description=
(f"Run Pylint on {pluralize(len(partition.field_sets), 'target')}: "
f"{address_references}."),
),
)
return LintResult.from_fallible_process_result(result,
linter_name="Pylint")
async def flake8_lint(
field_sets: Flake8FieldSets,
flake8: Flake8,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if flake8.options.skip:
return LintResult.noop()
# NB: Flake8 output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://flake8.pycqa.org/en/latest/user/invocation.html.
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(field_set.compatibility for field_set in field_sets), python_setup)
requirements_pex_request = Get[Pex](PexRequest(
output_filename="flake8.pex",
requirements=PexRequirements(flake8.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=flake8.get_entry_point(),
))
config_path: Optional[str] = flake8.options.config
config_snapshot_request = Get[Snapshot](PathGlobs(
globs=[config_path] if config_path else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--flake8-config`",
))
all_source_files_request = Get[SourceFiles](AllSourceFilesRequest(
field_set.sources for field_set in field_sets))
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest(
(field_set.sources, field_set.origin) for field_set in field_sets))
requirements_pex, config_snapshot, all_source_files, specified_source_files = cast(
Tuple[Pex, Snapshot, SourceFiles, SourceFiles],
await MultiGet([
requirements_pex_request,
config_snapshot_request,
all_source_files_request,
specified_source_files_request,
]),
)
input_digest = await Get[Digest](MergeDigests(
(all_source_files.snapshot.digest, requirements_pex.digest,
config_snapshot.digest)))
address_references = ", ".join(
sorted(field_set.address.reference() for field_set in field_sets))
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./flake8.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
flake8=flake8),
input_digest=input_digest,
description=
f"Run Flake8 on {pluralize(len(field_sets), 'target')}: {address_references}.",
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result,
linter_name="Flake8")
async def run_shellcheck(request: ShellcheckRequest,
shellcheck: Shellcheck) -> LintResults:
if shellcheck.options.skip:
return LintResults([], linter_name="Shellcheck")
# Shellcheck looks at direct dependencies to make sure that every symbol is defined, so we must
# include those in the run.
all_dependencies = await MultiGet(
Get(Targets, DependenciesRequest(field_set.dependencies))
for field_set in request.field_sets)
# Now that we have all dependencies, we flatten the results into a single list of `BashSources`
# fields and we filter out all targets without a `BashSources` field registered because those
# dependencies are irrelevant to Shellcheck.
dependencies_sources_fields = [
tgt[BashSources] for dependencies in all_dependencies
for tgt in dependencies if tgt.has_field(BashSources)
]
sources_request = Get(
SourceFiles,
SourceFilesRequest([
*(field_set.sources for field_set in request.field_sets),
*dependencies_sources_fields,
]),
)
download_shellcheck_request = Get(
DownloadedExternalTool,
ExternalToolRequest,
shellcheck.get_request(Platform.current),
)
# If the user specified `--shellcheck-config`, we search for the file they specified with
# `PathGlobs` to include it in the `input_digest`. We error if the file cannot be found.
# See https://www.pantsbuild.org/v2.0/docs/rules-api-file-system.
config_digest_request = Get(
Digest,
PathGlobs(
globs=[shellcheck.options.config]
if shellcheck.options.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--shellcheck-config`",
),
)
sources, downloaded_shellcheck, config_digest = await MultiGet(
sources_request, download_shellcheck_request, config_digest_request)
# The Process needs one single `Digest`, so we merge everything together. See
# https://www.pantsbuild.org/v2.0/docs/rules-api-file-system.
input_digest = await Get(
Digest,
MergeDigests((sources.snapshot.digest, downloaded_shellcheck.digest,
config_digest)),
)
# We use `FallibleProcessResult`, rather than `ProcessResult`, because we're okay with the
# Process failing.
process_result = await Get(
FallibleProcessResult,
Process(
argv=[
downloaded_shellcheck.exe,
*shellcheck.options.args,
*sources.snapshot.files,
],
input_digest=input_digest,
description=
f"Run Shellcheck on {pluralize(len(request.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
result = LintResult.from_fallible_process_result(process_result)
return LintResults([result], linter_name="Shellcheck")
async def pylint_lint_partition(partition: PylintPartition,
pylint: Pylint) -> LintResult:
requirements_pex_get = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(field_set.address for field_set in partition.field_sets),
# NB: These constraints must be identical to the other PEXes. Otherwise, we risk using
# a different version for the requirements than the other two PEXes, which can result
# in a PEX runtime error about missing dependencies.
hardcoded_interpreter_constraints=partition.
interpreter_constraints,
internal_only=True,
direct_deps_only=True,
),
)
plugin_requirements = PexRequirements.create_from_requirement_fields(
plugin_tgt[PythonRequirementsField]
for plugin_tgt in partition.plugin_targets
if plugin_tgt.has_field(PythonRequirementsField))
pylint_pex_get = Get(
Pex,
PexRequest(
output_filename="pylint.pex",
internal_only=True,
requirements=PexRequirements(
[*pylint.all_requirements, *plugin_requirements]),
interpreter_constraints=partition.interpreter_constraints,
),
)
prepare_plugin_sources_get = Get(
StrippedPythonSourceFiles,
PythonSourceFilesRequest(partition.plugin_targets))
prepare_python_sources_get = Get(
PythonSourceFiles,
PythonSourceFilesRequest(partition.targets_with_dependencies))
field_set_sources_get = Get(
SourceFiles,
SourceFilesRequest(field_set.sources
for field_set in partition.field_sets))
(
pylint_pex,
requirements_pex,
prepared_plugin_sources,
prepared_python_sources,
field_set_sources,
) = await MultiGet(
pylint_pex_get,
requirements_pex_get,
prepare_plugin_sources_get,
prepare_python_sources_get,
field_set_sources_get,
)
pylint_runner_pex, config_files = await MultiGet(
Get(
VenvPex,
PexRequest(
output_filename="pylint_runner.pex",
interpreter_constraints=partition.interpreter_constraints,
main=pylint.main,
internal_only=True,
pex_path=[pylint_pex, requirements_pex],
),
),
Get(ConfigFiles, ConfigFilesRequest,
pylint.config_request(field_set_sources.snapshot.dirs)),
)
prefixed_plugin_sources = (await Get(
Digest,
AddPrefix(
prepared_plugin_sources.stripped_source_files.snapshot.digest,
"__plugins"),
) if pylint.source_plugins else EMPTY_DIGEST)
pythonpath = list(prepared_python_sources.source_roots)
if pylint.source_plugins:
# NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must
# point to the plugin's directory, rather than to a parent's directory, because
# `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but
# not `path.to.plugin`. (This means users must have specified the parent directory as a
# source root.)
pythonpath.append("__plugins")
input_digest = await Get(
Digest,
MergeDigests((
config_files.snapshot.digest,
prefixed_plugin_sources,
prepared_python_sources.source_files.snapshot.digest,
)),
)
result = await Get(
FallibleProcessResult,
VenvPexProcess(
pylint_runner_pex,
argv=generate_argv(field_set_sources, pylint),
input_digest=input_digest,
extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},
description=
f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)))
async def pylint_lint_partition(
partition: PylintPartition,
pylint: Pylint,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
# We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies.
# Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`.
plugin_requirements = PexRequirements.create_from_requirement_fields(
plugin_tgt[PythonRequirementsField]
for plugin_tgt in partition.plugin_targets
if plugin_tgt.has_field(PythonRequirementsField))
target_requirements = PexRequirements.create_from_requirement_fields(
tgt[PythonRequirementsField]
for tgt in partition.targets_with_dependencies
if tgt.has_field(PythonRequirementsField))
pylint_pex_request = Get[Pex](PexRequest(
output_filename="pylint.pex",
requirements=PexRequirements(
[*pylint.get_requirement_specs(), *plugin_requirements]),
interpreter_constraints=partition.interpreter_constraints,
entry_point=pylint.get_entry_point(),
))
requirements_pex_request = Get[Pex](PexRequest(
output_filename="requirements.pex",
requirements=target_requirements,
interpreter_constraints=partition.interpreter_constraints,
))
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any Pylint transitive requirements will shadow corresponding user
# requirements, which could lead to problems.
pylint_runner_pex_args = [
"--pex-path", ":".join(["pylint.pex", "requirements.pex"])
]
if pylint.source_plugins:
# NB: See below for why we set PYTHONPATH to load source plugins. This setting is necessary
# for PEX to pick up the PYTHONPATH value.
pylint_runner_pex_args.append("--inherit-path=fallback")
pylint_runner_pex_request = Get[Pex](PexRequest(
output_filename="pylint_runner.pex",
entry_point=pylint.get_entry_point(),
interpreter_constraints=partition.interpreter_constraints,
additional_args=pylint_runner_pex_args,
))
config_snapshot_request = Get[Snapshot](PathGlobs(
globs=[pylint.config] if pylint.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
))
prepare_plugin_sources_request = Get[ImportablePythonSources](
Targets, partition.plugin_targets)
prepare_python_sources_request = Get[ImportablePythonSources](
Targets, partition.targets_with_dependencies)
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest(
((field_set.sources, field_set.origin)
for field_set in partition.field_sets),
strip_source_roots=True,
))
(
pylint_pex,
requirements_pex,
pylint_runner_pex,
config_snapshot,
prepared_plugin_sources,
prepared_python_sources,
specified_source_files,
) = await MultiGet(
pylint_pex_request,
requirements_pex_request,
pylint_runner_pex_request,
config_snapshot_request,
prepare_plugin_sources_request,
prepare_python_sources_request,
specified_source_files_request,
)
prefixed_plugin_sources = (await Get[Digest](AddPrefix(
prepared_plugin_sources.snapshot.digest, "__plugins"))
if pylint.source_plugins else EMPTY_DIGEST)
input_digest = await Get[Digest](MergeDigests((
pylint_pex.digest,
requirements_pex.digest,
pylint_runner_pex.digest,
config_snapshot.digest,
prefixed_plugin_sources,
prepared_python_sources.snapshot.digest,
)), )
address_references = ", ".join(
sorted(field_set.address.reference()
for field_set in partition.field_sets))
process = pylint_runner_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./pylint_runner.pex",
# NB: Pylint source plugins must be explicitly loaded via PYTHONPATH. The value must
# point to the plugin's directory, rather than to a parent's directory, because
# `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but
# not `path.to.plugin`. (This means users must have specified the parent directory as a
# source root.)
env={"PYTHONPATH": "./__plugins"} if pylint.source_plugins else None,
pex_args=generate_args(specified_source_files=specified_source_files,
pylint=pylint),
input_digest=input_digest,
description=
(f"Run Pylint on {pluralize(len(partition.field_sets), 'target')}: {address_references}."
),
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result,
linter_name="Pylint")
async def shfmt_lint(request: ShfmtRequest, shfmt: Shfmt) -> LintResults:
if shfmt.skip:
return LintResults([], linter_name=request.name)
setup = await Get(Setup, SetupRequest(request, check_only=True))
result = await Get(FallibleProcessResult, Process, setup.process)
return LintResults([LintResult.from_fallible_process_result(result)], linter_name=request.name)
async def pylint_lint_partition(
partition: PylintPartition, pylint: Pylint, first_party_plugins: PylintFirstPartyPlugins
) -> LintResult:
requirements_pex_get = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(field_set.address for field_set in partition.field_sets),
# NB: These constraints must be identical to the other PEXes. Otherwise, we risk using
# a different version for the requirements than the other two PEXes, which can result
# in a PEX runtime error about missing dependencies.
hardcoded_interpreter_constraints=partition.interpreter_constraints,
internal_only=True,
direct_deps_only=True,
),
)
pylint_pex_get = Get(
Pex,
PexRequest(
output_filename="pylint.pex",
internal_only=True,
requirements=pylint.pex_requirements(
extra_requirements=first_party_plugins.requirement_strings,
),
interpreter_constraints=partition.interpreter_constraints,
),
)
prepare_python_sources_get = Get(
PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies)
)
field_set_sources_get = Get(
SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)
)
pylint_pex, requirements_pex, prepared_python_sources, field_set_sources = await MultiGet(
pylint_pex_get,
requirements_pex_get,
prepare_python_sources_get,
field_set_sources_get,
)
pylint_runner_pex, config_files = await MultiGet(
Get(
VenvPex,
PexRequest(
output_filename="pylint_runner.pex",
interpreter_constraints=partition.interpreter_constraints,
main=pylint.main,
internal_only=True,
pex_path=[pylint_pex, requirements_pex],
),
),
Get(
ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs)
),
)
pythonpath = list(prepared_python_sources.source_roots)
if first_party_plugins:
pythonpath.append(first_party_plugins.PREFIX)
input_digest = await Get(
Digest,
MergeDigests(
(
config_files.snapshot.digest,
first_party_plugins.sources_digest,
prepared_python_sources.source_files.snapshot.digest,
)
),
)
result = await Get(
FallibleProcessResult,
VenvPexProcess(
pylint_runner_pex,
argv=generate_argv(field_set_sources, pylint),
input_digest=input_digest,
extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},
description=f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
return LintResult.from_fallible_process_result(
result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints))
)
async def black_lint(configs: BlackConfigurations, black: Black) -> LintResult:
if black.options.skip:
return LintResult.noop()
setup = await Get[Setup](SetupRequest(configs, check_only=True))
result = await Get[FallibleProcessResult](Process, setup.process)
return LintResult.from_fallible_process_result(result)
async def flake8_lint_partition(partition: Flake8Partition, flake8: Flake8,
lint_subsystem: LintSubsystem) -> LintResult:
requirements_pex_request = Get(
Pex,
PexRequest(
output_filename="flake8.pex",
requirements=PexRequirements(flake8.all_requirements),
interpreter_constraints=(partition.interpreter_constraints
or PexInterpreterConstraints(
flake8.interpreter_constraints)),
entry_point=flake8.entry_point,
),
)
config_digest_request = Get(
Digest,
PathGlobs(
globs=[flake8.config] if flake8.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--flake8-config`",
),
)
source_files_request = Get(
SourceFiles,
SourceFilesRequest(field_set.sources
for field_set in partition.field_sets))
requirements_pex, config_digest, source_files = await MultiGet(
requirements_pex_request, config_digest_request, source_files_request)
input_digest = await Get(
Digest,
MergeDigests((source_files.snapshot.digest, requirements_pex.digest,
config_digest)),
)
address_references = ", ".join(
sorted(field_set.address.spec for field_set in partition.field_sets))
report_path = (lint_subsystem.reports_dir /
"flake8_report.txt" if lint_subsystem.reports_dir else None)
result = await Get(
FallibleProcessResult,
PexProcess(
requirements_pex,
argv=generate_args(
source_files=source_files,
flake8=flake8,
output_file=report_path.name if report_path else None,
),
input_digest=input_digest,
output_files=(report_path.name, ) if report_path else None,
description=
(f"Run Flake8 on {pluralize(len(partition.field_sets), 'target')}: "
f"{address_references}."),
),
)
results_file = None
if report_path:
report_file_snapshot = await Get(
Snapshot,
DigestSubset(result.output_digest, PathGlobs([report_path.name])))
if len(report_file_snapshot.files) != 1:
raise Exception(
f"Unexpected report file snapshot: {report_file_snapshot.files}"
)
results_file = LintResultFile(output_path=report_path,
digest=report_file_snapshot.digest)
return LintResult.from_fallible_process_result(result,
linter_name="Flake8",
results_file=results_file)
async def pylint_lint_partition(partition: PylintPartition, pylint: Pylint) -> LintResult:
requirements_pex_request = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
(field_set.address for field_set in partition.field_sets),
# NB: These constraints must be identical to the other PEXes. Otherwise, we risk using
# a different version for the requirements than the other two PEXes, which can result
# in a PEX runtime error about missing dependencies.
hardcoded_interpreter_constraints=partition.interpreter_constraints,
internal_only=True,
direct_deps_only=True,
),
)
plugin_requirements = PexRequirements.create_from_requirement_fields(
plugin_tgt[PythonRequirementsField]
for plugin_tgt in partition.plugin_targets
if plugin_tgt.has_field(PythonRequirementsField)
)
# Right now any Pylint transitive requirements will shadow corresponding user
# requirements, which could lead to problems.
pylint_pex_request = Get(
Pex,
PexRequest(
output_filename="pylint.pex",
internal_only=True,
requirements=PexRequirements([*pylint.all_requirements, *plugin_requirements]),
entry_point=pylint.entry_point,
interpreter_constraints=partition.interpreter_constraints,
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
additional_args=("--pex-path", requirements_pex_request.input.output_filename),
),
)
config_digest_request = Get(
Digest,
PathGlobs(
globs=[pylint.config] if pylint.config else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
),
)
prepare_plugin_sources_request = Get(
StrippedPythonSourceFiles, PythonSourceFilesRequest(partition.plugin_targets)
)
prepare_python_sources_request = Get(
PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies)
)
field_set_sources_request = Get(
SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)
)
(
pylint_pex,
requirements_pex,
config_digest,
prepared_plugin_sources,
prepared_python_sources,
field_set_sources,
) = await MultiGet(
pylint_pex_request,
requirements_pex_request,
config_digest_request,
prepare_plugin_sources_request,
prepare_python_sources_request,
field_set_sources_request,
)
prefixed_plugin_sources = (
await Get(
Digest,
AddPrefix(prepared_plugin_sources.stripped_source_files.snapshot.digest, "__plugins"),
)
if pylint.source_plugins
else EMPTY_DIGEST
)
pythonpath = list(prepared_python_sources.source_roots)
if pylint.source_plugins:
# NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must
# point to the plugin's directory, rather than to a parent's directory, because
# `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but
# not `path.to.plugin`. (This means users must have specified the parent directory as a
# source root.)
pythonpath.append("__plugins")
input_digest = await Get(
Digest,
MergeDigests(
(
pylint_pex.digest,
requirements_pex.digest,
config_digest,
prefixed_plugin_sources,
prepared_python_sources.source_files.snapshot.digest,
)
),
)
result = await Get(
FallibleProcessResult,
PexProcess(
pylint_pex,
argv=generate_args(source_files=field_set_sources, pylint=pylint),
input_digest=input_digest,
extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)},
description=f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.",
level=LogLevel.DEBUG,
),
)
return LintResult.from_fallible_process_result(
result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints))
)
async def bandit_lint_partition(partition: BanditPartition, bandit: Bandit,
lint_subsystem: LintSubsystem) -> LintResult:
bandit_pex_get = Get(
VenvPex,
PexRequest(
output_filename="bandit.pex",
internal_only=True,
requirements=PexRequirements(bandit.all_requirements),
interpreter_constraints=partition.interpreter_constraints,
main=bandit.main,
),
)
config_files_get = Get(ConfigFiles, ConfigFilesRequest,
bandit.config_request)
source_files_get = Get(
SourceFiles,
SourceFilesRequest(field_set.sources
for field_set in partition.field_sets))
bandit_pex, config_files, source_files = await MultiGet(
bandit_pex_get, config_files_get, source_files_get)
input_digest = await Get(
Digest,
MergeDigests(
(source_files.snapshot.digest, config_files.snapshot.digest)))
report_file_name = "bandit_report.txt" if lint_subsystem.reports_dir else None
result = await Get(
FallibleProcessResult,
VenvPexProcess(
bandit_pex,
argv=generate_argv(source_files,
bandit,
report_file_name=report_file_name),
input_digest=input_digest,
description=
f"Run Bandit on {pluralize(len(partition.field_sets), 'file')}.",
output_files=(report_file_name, ) if report_file_name else None,
level=LogLevel.DEBUG,
),
)
report = None
if report_file_name:
report_digest = await Get(
Digest,
DigestSubset(
result.output_digest,
PathGlobs(
[report_file_name],
glob_match_error_behavior=GlobMatchErrorBehavior.warn,
description_of_origin="Bandit report file",
),
),
)
report = LintReport(report_file_name, report_digest)
return LintResult.from_fallible_process_result(
result,
partition_description=str(
sorted(str(c) for c in partition.interpreter_constraints)),
report=report,
)
