async def flake8_lint_partition(partition: Flake8Partition, flake8: Flake8) -> LintResult: flake8_pex_get = Get( VenvPex, PexRequest( output_filename="flake8.pex", internal_only=True, requirements=flake8.pex_requirements(), interpreter_constraints=partition.interpreter_constraints, main=flake8.main, ), ) config_files_get = Get(ConfigFiles, ConfigFilesRequest, flake8.config_request) source_files_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets) ) # Ensure that the empty report dir exists. report_directory_digest_get = Get(Digest, CreateDigest([Directory(REPORT_DIR)])) flake8_pex, config_files, report_directory, source_files = await MultiGet( flake8_pex_get, config_files_get, report_directory_digest_get, source_files_get ) input_digest = await Get( Digest, MergeDigests( (source_files.snapshot.digest, config_files.snapshot.digest, report_directory) ), ) result = await Get( FallibleProcessResult, VenvPexProcess( flake8_pex, argv=generate_argv(source_files, flake8), input_digest=input_digest, output_directories=(REPORT_DIR,), description=f"Run Flake8 on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR)) return LintResult.from_fallible_process_result( result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)), report=report, )
async def google_java_format_lint( request: GoogleJavaFormatRequest, tool: GoogleJavaFormatSubsystem) -> LintResults: if tool.skip: return LintResults([], linter_name=request.name) setup = await Get(Setup, SetupRequest(request, check_only=True)) result = await Get(FallibleProcessResult, JvmProcess, setup.process) lint_result = LintResult.from_fallible_process_result(result) if lint_result.exit_code == 0 and lint_result.stdout.strip() != "": # Note: The formetter returns success even if it would have reformatted the files. # When this occurs, convert the LintResult into a failure. lint_result = dataclasses.replace( lint_result, exit_code=1, stdout= f"The following Java files require formatting:\n{lint_result.stdout}\n", ) return LintResults([lint_result], linter_name=request.name)
async def run_hadolint(request: HadolintRequest, hadolint: Hadolint) -> LintResults: if hadolint.skip: return LintResults([], linter_name="Hadolint") downloaded_hadolint, sources, config_files = await MultiGet( Get(DownloadedExternalTool, ExternalToolRequest, hadolint.get_request(Platform.current)), Get( SourceFiles, SourceFilesRequest( [field_set.sources for field_set in request.field_sets], for_sources_types=(DockerImageSources, ), enable_codegen=True, ), ), Get(ConfigFiles, ConfigFilesRequest, hadolint.config_request()), ) input_digest = await Get( Digest, MergeDigests(( sources.snapshot.digest, downloaded_hadolint.digest, config_files.snapshot.digest, )), ) process_result = await Get( FallibleProcessResult, Process( argv=[downloaded_hadolint.exe, *generate_argv(sources, hadolint)], input_digest=input_digest, description= f"Run `hadolint` on {pluralize(len(sources.files), 'Dockerfile')}.", level=LogLevel.DEBUG, ), ) return LintResults( [LintResult.from_fallible_process_result(process_result)], linter_name="hadolint")
async def run_shellcheck(request: ShellcheckRequest, shellcheck: Shellcheck) -> LintResults: if shellcheck.skip: return LintResults([], linter_name="Shellcheck") # Shellcheck looks at direct dependencies to make sure that every symbol is defined, so we must # include those in the run. all_dependencies = await MultiGet( Get(Targets, DependenciesRequest(field_set.dependencies)) for field_set in request.field_sets) direct_sources_get = Get( SourceFiles, SourceFilesRequest( (field_set.sources for field_set in request.field_sets), for_sources_types=(ShellSourcesField, ), enable_codegen=True, ), ) dependency_sources_get = Get( SourceFiles, SourceFilesRequest( (tgt.get(Sources) for dependencies in all_dependencies for tgt in dependencies), for_sources_types=(ShellSourcesField, ), enable_codegen=True, ), ) download_shellcheck_get = Get(DownloadedExternalTool, ExternalToolRequest, shellcheck.get_request(Platform.current)) direct_sources, dependency_sources, downloaded_shellcheck = await MultiGet( direct_sources_get, dependency_sources_get, download_shellcheck_get) config_files = await Get( ConfigFiles, ConfigFilesRequest, shellcheck.config_request(direct_sources.snapshot.dirs)) input_digest = await Get( Digest, MergeDigests(( direct_sources.snapshot.digest, dependency_sources.snapshot.digest, downloaded_shellcheck.digest, config_files.snapshot.digest, )), ) process_result = await Get( FallibleProcessResult, Process( argv=[ downloaded_shellcheck.exe, *shellcheck.args, *direct_sources.snapshot.files ], input_digest=input_digest, description= f"Run Shellcheck on {pluralize(len(request.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) result = LintResult.from_fallible_process_result(process_result) return LintResults([result], linter_name="Shellcheck")
async def bandit_lint( field_sets: BanditFieldSets, bandit: Bandit, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if bandit.options.skip: return LintResult.noop() # NB: Bandit output depends upon which Python interpreter version it's run with. See # https://github.com/PyCQA/bandit#under-which-version-of-python-should-i-install-bandit. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (field_set.compatibility for field_set in field_sets), python_setup=python_setup) requirements_pex_request = Get[Pex](PexRequest( output_filename="bandit.pex", requirements=PexRequirements(bandit.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=bandit.get_entry_point(), )) config_path: Optional[str] = bandit.options.config config_snapshot_request = Get[Snapshot](PathGlobs( globs=tuple([config_path] if config_path else []), glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--bandit-config`", )) all_source_files_request = Get[SourceFiles](AllSourceFilesRequest( field_set.sources for field_set in field_sets)) specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest( (field_set.sources, field_set.origin) for field_set in field_sets)) requirements_pex, config_snapshot, all_source_files, specified_source_files = cast( Tuple[Pex, Snapshot, SourceFiles, SourceFiles], await MultiGet([ requirements_pex_request, config_snapshot_request, all_source_files_request, specified_source_files_request, ]), ) input_digest = await Get[Digest](MergeDigests( (all_source_files.snapshot.digest, requirements_pex.digest, config_snapshot.digest))) address_references = ", ".join( sorted(field_set.address.reference() for field_set in field_sets)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./bandit.pex", pex_args=generate_args(specified_source_files=specified_source_files, bandit=bandit), input_digest=input_digest, description= f"Run Bandit on {pluralize(len(field_sets), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result, linter_name="Bandit")
async def pylint_lint( configs: PylintConfigurations, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if pylint.options.skip: return LintResult.noop() # Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it # doesn't lint those direct dependencies nor does it care about transitive dependencies. addresses = [] for config in configs: addresses.append(config.address) addresses.extend(config.dependencies.value or ()) targets = await Get[Targets](Addresses(addresses)) chrooted_python_sources = await Get[ImportablePythonSources](Targets, targets) # NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (config.compatibility for config in configs), python_setup) requirements_pex = await Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements(pylint.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=pylint.get_entry_point(), )) config_path: Optional[str] = pylint.options.config config_snapshot = await Get[Snapshot](PathGlobs( globs=tuple([config_path] if config_path else []), glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) merged_input_files = await Get[Digest](DirectoriesToMerge(directories=( requirements_pex.directory_digest, config_snapshot.directory_digest, chrooted_python_sources.snapshot.directory_digest, )), ) specified_source_files = await Get[SourceFiles]( SpecifiedSourceFilesRequest( ((config.sources, config.origin) for config in configs), strip_source_roots=True)) address_references = ", ".join( sorted(config.address.reference() for config in configs)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./pylint.pex", pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_files=merged_input_files, description= f"Run Pylint on {pluralize(len(configs), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result)
async def pylint_lint_partition( partition: PylintPartition, pylint: Pylint, first_party_plugins: PylintFirstPartyPlugins) -> LintResult: requirements_pex_get = Get( Pex, RequirementsPexRequest( (t.address for t in partition.root_targets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, ), ) pylint_pex_get = Get( Pex, PexRequest, pylint.to_pex_request( interpreter_constraints=partition.interpreter_constraints, extra_requirements=first_party_plugins.requirement_strings, ), ) prepare_python_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) field_set_sources_get = Get( SourceFiles, SourceFilesRequest(t[PythonSourceField] for t in partition.root_targets)) # Ensure that the empty report dir exists. report_directory_digest_get = Get(Digest, CreateDigest([Directory(REPORT_DIR)])) ( pylint_pex, requirements_pex, prepared_python_sources, field_set_sources, report_directory, ) = await MultiGet( pylint_pex_get, requirements_pex_get, prepare_python_sources_get, field_set_sources_get, report_directory_digest_get, ) pylint_runner_pex, config_files = await MultiGet( Get( VenvPex, VenvPexRequest( PexRequest( output_filename="pylint_runner.pex", interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, pex_path=[pylint_pex, requirements_pex], ), # TODO(John Sirois): Remove this (change to the default of symlinks) when we can # upgrade to a version of Pylint with https://github.com/PyCQA/pylint/issues/1470 # resolved. site_packages_copies=True, ), ), Get(ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs)), ) pythonpath = list(prepared_python_sources.source_roots) if first_party_plugins: pythonpath.append(first_party_plugins.PREFIX) input_digest = await Get( Digest, MergeDigests(( config_files.snapshot.digest, first_party_plugins.sources_digest, prepared_python_sources.source_files.snapshot.digest, report_directory, )), ) result = await Get( FallibleProcessResult, VenvPexProcess( pylint_runner_pex, argv=generate_argv(field_set_sources, pylint), input_digest=input_digest, output_directories=(REPORT_DIR, ), extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, concurrency_available=len(partition.root_targets), description= f"Run Pylint on {pluralize(len(partition.root_targets), 'file')}.", level=LogLevel.DEBUG, ), ) report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR)) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)), report=report, )
async def pylint_lint( field_sets: PylintFieldSets, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if pylint.skip: return LintResult.noop() # Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it # doesn't lint those direct dependencies nor does it care about transitive dependencies. addresses_with_dependencies = [] for field_set in field_sets: addresses_with_dependencies.append(field_set.address) addresses_with_dependencies.extend(field_set.dependencies.value or ()) targets = await Get[Targets](Addresses(addresses_with_dependencies)) # NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (field_set.compatibility for field_set in field_sets), python_setup) # We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies. # Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`. pylint_pex_request = Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements(pylint.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=pylint.get_entry_point(), )) requirements_pex_request = Get[Pex](PexRequest( output_filename="requirements.pex", requirements=PexRequirements.create_from_requirement_fields( tgt[PythonRequirementsField] for tgt in targets if tgt.has_field(PythonRequirementsField)), interpreter_constraints=interpreter_constraints, )) pylint_runner_pex_request = Get[Pex]( PexRequest( output_filename="pylint_runner.pex", entry_point=pylint.get_entry_point(), interpreter_constraints=interpreter_constraints, additional_args=( "--pex-path", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any Pylint transitive requirements will shadow corresponding user # requirements which could lead to problems. ":".join(["pylint.pex", "requirements.pex"]), ), )) config_snapshot_request = Get[Snapshot](PathGlobs( globs=[pylint.config] if pylint.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) prepare_python_sources_request = Get[ImportablePythonSources](Targets, targets) specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest( ((field_set.sources, field_set.origin) for field_set in field_sets), strip_source_roots=True, )) ( pylint_pex, requirements_pex, pylint_runner_pex, config_snapshot, prepared_python_sources, specified_source_files, ) = cast( Tuple[Pex, Pex, Pex, Snapshot, ImportablePythonSources, SourceFiles], await MultiGet([ pylint_pex_request, requirements_pex_request, pylint_runner_pex_request, config_snapshot_request, prepare_python_sources_request, specified_source_files_request, ]), ) input_digest = await Get[Digest](MergeDigests(( pylint_pex.digest, requirements_pex.digest, pylint_runner_pex.digest, config_snapshot.digest, prepared_python_sources.snapshot.digest, )), ) address_references = ", ".join( sorted(field_set.address.reference() for field_set in field_sets)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./pylint_runner.pex", pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_digest=input_digest, description= f"Run Pylint on {pluralize(len(field_sets), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result, linter_name="Pylint")
async def flake8_lint_partition( partition: Flake8Partition, flake8: Flake8, lint_subsystem: LintSubsystem ) -> LintResult: requirements_pex_request = Get( Pex, PexRequest( output_filename="flake8.pex", internal_only=True, requirements=PexRequirements(flake8.all_requirements), interpreter_constraints=( partition.interpreter_constraints or PexInterpreterConstraints(flake8.interpreter_constraints) ), entry_point=flake8.entry_point, ), ) config_digest_request = Get( Digest, PathGlobs( globs=[flake8.config] if flake8.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--flake8-config`", ), ) source_files_request = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets) ) requirements_pex, config_digest, source_files = await MultiGet( requirements_pex_request, config_digest_request, source_files_request ) input_digest = await Get( Digest, MergeDigests((source_files.snapshot.digest, requirements_pex.digest, config_digest)), ) report_file_name = "flake8_report.txt" if lint_subsystem.reports_dir else None result = await Get( FallibleProcessResult, PexProcess( requirements_pex, argv=generate_args( source_files=source_files, flake8=flake8, report_file_name=report_file_name ), input_digest=input_digest, output_files=(report_file_name,) if report_file_name else None, description=f"Run Flake8 on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) report = None if report_file_name: report_digest = await Get( Digest, DigestSubset( result.output_digest, PathGlobs( [report_file_name], glob_match_error_behavior=GlobMatchErrorBehavior.warn, description_of_origin="Flake8 report file", ), ), ) report = LintReport(report_file_name, report_digest) return LintResult.from_fallible_process_result( result, partition_description=str(sorted(partition.interpreter_constraints)), report=report )
async def pylint_lint_partition(partition: PylintPartition, pylint: Pylint) -> LintResult: # We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies. # Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`. plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in partition.plugin_targets if plugin_tgt.has_field(PythonRequirementsField)) target_requirements = PexRequirements.create_from_requirement_fields( tgt[PythonRequirementsField] for tgt in partition.targets_with_dependencies if tgt.has_field(PythonRequirementsField)) pylint_pex_request = Get( Pex, PexRequest( output_filename="pylint.pex", requirements=PexRequirements( [*pylint.all_requirements, *plugin_requirements]), interpreter_constraints=partition.interpreter_constraints, ), ) requirements_pex_request = Get( Pex, PexRequest( output_filename="requirements.pex", requirements=target_requirements, interpreter_constraints=partition.interpreter_constraints, ), ) # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any Pylint transitive requirements will shadow corresponding user # requirements, which could lead to problems. pylint_runner_pex_args = [ "--pex-path", ":".join(["pylint.pex", "requirements.pex"]) ] pylint_runner_pex_request = Get( Pex, PexRequest( output_filename="pylint_runner.pex", entry_point=pylint.entry_point, interpreter_constraints=partition.interpreter_constraints, additional_args=pylint_runner_pex_args, ), ) config_digest_request = Get( Digest, PathGlobs( globs=[pylint.config] if pylint.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", ), ) prepare_plugin_sources_request = Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(partition.plugin_targets), ) prepare_python_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies), ) field_set_sources_request = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets), ) ( pylint_pex, requirements_pex, pylint_runner_pex, config_digest, prepared_plugin_sources, prepared_python_sources, field_set_sources, ) = await MultiGet( pylint_pex_request, requirements_pex_request, pylint_runner_pex_request, config_digest_request, prepare_plugin_sources_request, prepare_python_sources_request, field_set_sources_request, ) prefixed_plugin_sources = (await Get( Digest, AddPrefix( prepared_plugin_sources.stripped_source_files.snapshot.digest, "__plugins"), ) if pylint.source_plugins else EMPTY_DIGEST) pythonpath = list(prepared_python_sources.source_roots) if pylint.source_plugins: # NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must # point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) pythonpath.append("__plugins") input_digest = await Get( Digest, MergeDigests(( pylint_pex.digest, requirements_pex.digest, pylint_runner_pex.digest, config_digest, prefixed_plugin_sources, prepared_python_sources.source_files.snapshot.digest, )), ) address_references = ", ".join( sorted(field_set.address.spec for field_set in partition.field_sets)) result = await Get( FallibleProcessResult, PexProcess( pylint_runner_pex, argv=generate_args(source_files=field_set_sources, pylint=pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description= (f"Run Pylint on {pluralize(len(partition.field_sets), 'target')}: " f"{address_references}."), ), ) return LintResult.from_fallible_process_result(result, linter_name="Pylint")
async def flake8_lint( field_sets: Flake8FieldSets, flake8: Flake8, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if flake8.options.skip: return LintResult.noop() # NB: Flake8 output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://flake8.pycqa.org/en/latest/user/invocation.html. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (field_set.compatibility for field_set in field_sets), python_setup) requirements_pex_request = Get[Pex](PexRequest( output_filename="flake8.pex", requirements=PexRequirements(flake8.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=flake8.get_entry_point(), )) config_path: Optional[str] = flake8.options.config config_snapshot_request = Get[Snapshot](PathGlobs( globs=[config_path] if config_path else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--flake8-config`", )) all_source_files_request = Get[SourceFiles](AllSourceFilesRequest( field_set.sources for field_set in field_sets)) specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest( (field_set.sources, field_set.origin) for field_set in field_sets)) requirements_pex, config_snapshot, all_source_files, specified_source_files = cast( Tuple[Pex, Snapshot, SourceFiles, SourceFiles], await MultiGet([ requirements_pex_request, config_snapshot_request, all_source_files_request, specified_source_files_request, ]), ) input_digest = await Get[Digest](MergeDigests( (all_source_files.snapshot.digest, requirements_pex.digest, config_snapshot.digest))) address_references = ", ".join( sorted(field_set.address.reference() for field_set in field_sets)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./flake8.pex", pex_args=generate_args(specified_source_files=specified_source_files, flake8=flake8), input_digest=input_digest, description= f"Run Flake8 on {pluralize(len(field_sets), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result, linter_name="Flake8")
async def run_shellcheck(request: ShellcheckRequest, shellcheck: Shellcheck) -> LintResults: if shellcheck.options.skip: return LintResults([], linter_name="Shellcheck") # Shellcheck looks at direct dependencies to make sure that every symbol is defined, so we must # include those in the run. all_dependencies = await MultiGet( Get(Targets, DependenciesRequest(field_set.dependencies)) for field_set in request.field_sets) # Now that we have all dependencies, we flatten the results into a single list of `BashSources` # fields and we filter out all targets without a `BashSources` field registered because those # dependencies are irrelevant to Shellcheck. dependencies_sources_fields = [ tgt[BashSources] for dependencies in all_dependencies for tgt in dependencies if tgt.has_field(BashSources) ] sources_request = Get( SourceFiles, SourceFilesRequest([ *(field_set.sources for field_set in request.field_sets), *dependencies_sources_fields, ]), ) download_shellcheck_request = Get( DownloadedExternalTool, ExternalToolRequest, shellcheck.get_request(Platform.current), ) # If the user specified `--shellcheck-config`, we search for the file they specified with # `PathGlobs` to include it in the `input_digest`. We error if the file cannot be found. # See https://www.pantsbuild.org/v2.0/docs/rules-api-file-system. config_digest_request = Get( Digest, PathGlobs( globs=[shellcheck.options.config] if shellcheck.options.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--shellcheck-config`", ), ) sources, downloaded_shellcheck, config_digest = await MultiGet( sources_request, download_shellcheck_request, config_digest_request) # The Process needs one single `Digest`, so we merge everything together. See # https://www.pantsbuild.org/v2.0/docs/rules-api-file-system. input_digest = await Get( Digest, MergeDigests((sources.snapshot.digest, downloaded_shellcheck.digest, config_digest)), ) # We use `FallibleProcessResult`, rather than `ProcessResult`, because we're okay with the # Process failing. process_result = await Get( FallibleProcessResult, Process( argv=[ downloaded_shellcheck.exe, *shellcheck.options.args, *sources.snapshot.files, ], input_digest=input_digest, description= f"Run Shellcheck on {pluralize(len(request.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) result = LintResult.from_fallible_process_result(process_result) return LintResults([result], linter_name="Shellcheck")
async def pylint_lint_partition(partition: PylintPartition, pylint: Pylint) -> LintResult: requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (field_set.address for field_set in partition.field_sets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, direct_deps_only=True, ), ) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in partition.plugin_targets if plugin_tgt.has_field(PythonRequirementsField)) pylint_pex_get = Get( Pex, PexRequest( output_filename="pylint.pex", internal_only=True, requirements=PexRequirements( [*pylint.all_requirements, *plugin_requirements]), interpreter_constraints=partition.interpreter_constraints, ), ) prepare_plugin_sources_get = Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(partition.plugin_targets)) prepare_python_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies)) field_set_sources_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)) ( pylint_pex, requirements_pex, prepared_plugin_sources, prepared_python_sources, field_set_sources, ) = await MultiGet( pylint_pex_get, requirements_pex_get, prepare_plugin_sources_get, prepare_python_sources_get, field_set_sources_get, ) pylint_runner_pex, config_files = await MultiGet( Get( VenvPex, PexRequest( output_filename="pylint_runner.pex", interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, pex_path=[pylint_pex, requirements_pex], ), ), Get(ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs)), ) prefixed_plugin_sources = (await Get( Digest, AddPrefix( prepared_plugin_sources.stripped_source_files.snapshot.digest, "__plugins"), ) if pylint.source_plugins else EMPTY_DIGEST) pythonpath = list(prepared_python_sources.source_roots) if pylint.source_plugins: # NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must # point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) pythonpath.append("__plugins") input_digest = await Get( Digest, MergeDigests(( config_files.snapshot.digest, prefixed_plugin_sources, prepared_python_sources.source_files.snapshot.digest, )), ) result = await Get( FallibleProcessResult, VenvPexProcess( pylint_runner_pex, argv=generate_argv(field_set_sources, pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description= f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)))
async def pylint_lint_partition( partition: PylintPartition, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: # We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies. # Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`. plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in partition.plugin_targets if plugin_tgt.has_field(PythonRequirementsField)) target_requirements = PexRequirements.create_from_requirement_fields( tgt[PythonRequirementsField] for tgt in partition.targets_with_dependencies if tgt.has_field(PythonRequirementsField)) pylint_pex_request = Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements( [*pylint.get_requirement_specs(), *plugin_requirements]), interpreter_constraints=partition.interpreter_constraints, entry_point=pylint.get_entry_point(), )) requirements_pex_request = Get[Pex](PexRequest( output_filename="requirements.pex", requirements=target_requirements, interpreter_constraints=partition.interpreter_constraints, )) # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any Pylint transitive requirements will shadow corresponding user # requirements, which could lead to problems. pylint_runner_pex_args = [ "--pex-path", ":".join(["pylint.pex", "requirements.pex"]) ] if pylint.source_plugins: # NB: See below for why we set PYTHONPATH to load source plugins. This setting is necessary # for PEX to pick up the PYTHONPATH value. pylint_runner_pex_args.append("--inherit-path=fallback") pylint_runner_pex_request = Get[Pex](PexRequest( output_filename="pylint_runner.pex", entry_point=pylint.get_entry_point(), interpreter_constraints=partition.interpreter_constraints, additional_args=pylint_runner_pex_args, )) config_snapshot_request = Get[Snapshot](PathGlobs( globs=[pylint.config] if pylint.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) prepare_plugin_sources_request = Get[ImportablePythonSources]( Targets, partition.plugin_targets) prepare_python_sources_request = Get[ImportablePythonSources]( Targets, partition.targets_with_dependencies) specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest( ((field_set.sources, field_set.origin) for field_set in partition.field_sets), strip_source_roots=True, )) ( pylint_pex, requirements_pex, pylint_runner_pex, config_snapshot, prepared_plugin_sources, prepared_python_sources, specified_source_files, ) = await MultiGet( pylint_pex_request, requirements_pex_request, pylint_runner_pex_request, config_snapshot_request, prepare_plugin_sources_request, prepare_python_sources_request, specified_source_files_request, ) prefixed_plugin_sources = (await Get[Digest](AddPrefix( prepared_plugin_sources.snapshot.digest, "__plugins")) if pylint.source_plugins else EMPTY_DIGEST) input_digest = await Get[Digest](MergeDigests(( pylint_pex.digest, requirements_pex.digest, pylint_runner_pex.digest, config_snapshot.digest, prefixed_plugin_sources, prepared_python_sources.snapshot.digest, )), ) address_references = ", ".join( sorted(field_set.address.reference() for field_set in partition.field_sets)) process = pylint_runner_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./pylint_runner.pex", # NB: Pylint source plugins must be explicitly loaded via PYTHONPATH. The value must # point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) env={"PYTHONPATH": "./__plugins"} if pylint.source_plugins else None, pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_digest=input_digest, description= (f"Run Pylint on {pluralize(len(partition.field_sets), 'target')}: {address_references}." ), ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result, linter_name="Pylint")
async def shfmt_lint(request: ShfmtRequest, shfmt: Shfmt) -> LintResults: if shfmt.skip: return LintResults([], linter_name=request.name) setup = await Get(Setup, SetupRequest(request, check_only=True)) result = await Get(FallibleProcessResult, Process, setup.process) return LintResults([LintResult.from_fallible_process_result(result)], linter_name=request.name)
async def pylint_lint_partition( partition: PylintPartition, pylint: Pylint, first_party_plugins: PylintFirstPartyPlugins ) -> LintResult: requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (field_set.address for field_set in partition.field_sets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition.interpreter_constraints, internal_only=True, direct_deps_only=True, ), ) pylint_pex_get = Get( Pex, PexRequest( output_filename="pylint.pex", internal_only=True, requirements=pylint.pex_requirements( extra_requirements=first_party_plugins.requirement_strings, ), interpreter_constraints=partition.interpreter_constraints, ), ) prepare_python_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies) ) field_set_sources_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets) ) pylint_pex, requirements_pex, prepared_python_sources, field_set_sources = await MultiGet( pylint_pex_get, requirements_pex_get, prepare_python_sources_get, field_set_sources_get, ) pylint_runner_pex, config_files = await MultiGet( Get( VenvPex, PexRequest( output_filename="pylint_runner.pex", interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, pex_path=[pylint_pex, requirements_pex], ), ), Get( ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs) ), ) pythonpath = list(prepared_python_sources.source_roots) if first_party_plugins: pythonpath.append(first_party_plugins.PREFIX) input_digest = await Get( Digest, MergeDigests( ( config_files.snapshot.digest, first_party_plugins.sources_digest, prepared_python_sources.source_files.snapshot.digest, ) ), ) result = await Get( FallibleProcessResult, VenvPexProcess( pylint_runner_pex, argv=generate_argv(field_set_sources, pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description=f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return LintResult.from_fallible_process_result( result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)) )
async def black_lint(configs: BlackConfigurations, black: Black) -> LintResult: if black.options.skip: return LintResult.noop() setup = await Get[Setup](SetupRequest(configs, check_only=True)) result = await Get[FallibleProcessResult](Process, setup.process) return LintResult.from_fallible_process_result(result)
async def flake8_lint_partition(partition: Flake8Partition, flake8: Flake8, lint_subsystem: LintSubsystem) -> LintResult: requirements_pex_request = Get( Pex, PexRequest( output_filename="flake8.pex", requirements=PexRequirements(flake8.all_requirements), interpreter_constraints=(partition.interpreter_constraints or PexInterpreterConstraints( flake8.interpreter_constraints)), entry_point=flake8.entry_point, ), ) config_digest_request = Get( Digest, PathGlobs( globs=[flake8.config] if flake8.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--flake8-config`", ), ) source_files_request = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)) requirements_pex, config_digest, source_files = await MultiGet( requirements_pex_request, config_digest_request, source_files_request) input_digest = await Get( Digest, MergeDigests((source_files.snapshot.digest, requirements_pex.digest, config_digest)), ) address_references = ", ".join( sorted(field_set.address.spec for field_set in partition.field_sets)) report_path = (lint_subsystem.reports_dir / "flake8_report.txt" if lint_subsystem.reports_dir else None) result = await Get( FallibleProcessResult, PexProcess( requirements_pex, argv=generate_args( source_files=source_files, flake8=flake8, output_file=report_path.name if report_path else None, ), input_digest=input_digest, output_files=(report_path.name, ) if report_path else None, description= (f"Run Flake8 on {pluralize(len(partition.field_sets), 'target')}: " f"{address_references}."), ), ) results_file = None if report_path: report_file_snapshot = await Get( Snapshot, DigestSubset(result.output_digest, PathGlobs([report_path.name]))) if len(report_file_snapshot.files) != 1: raise Exception( f"Unexpected report file snapshot: {report_file_snapshot.files}" ) results_file = LintResultFile(output_path=report_path, digest=report_file_snapshot.digest) return LintResult.from_fallible_process_result(result, linter_name="Flake8", results_file=results_file)
async def pylint_lint_partition(partition: PylintPartition, pylint: Pylint) -> LintResult: requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (field_set.address for field_set in partition.field_sets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition.interpreter_constraints, internal_only=True, direct_deps_only=True, ), ) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in partition.plugin_targets if plugin_tgt.has_field(PythonRequirementsField) ) # Right now any Pylint transitive requirements will shadow corresponding user # requirements, which could lead to problems. pylint_pex_request = Get( Pex, PexRequest( output_filename="pylint.pex", internal_only=True, requirements=PexRequirements([*pylint.all_requirements, *plugin_requirements]), entry_point=pylint.entry_point, interpreter_constraints=partition.interpreter_constraints, # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 additional_args=("--pex-path", requirements_pex_request.input.output_filename), ), ) config_digest_request = Get( Digest, PathGlobs( globs=[pylint.config] if pylint.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", ), ) prepare_plugin_sources_request = Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(partition.plugin_targets) ) prepare_python_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies) ) field_set_sources_request = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets) ) ( pylint_pex, requirements_pex, config_digest, prepared_plugin_sources, prepared_python_sources, field_set_sources, ) = await MultiGet( pylint_pex_request, requirements_pex_request, config_digest_request, prepare_plugin_sources_request, prepare_python_sources_request, field_set_sources_request, ) prefixed_plugin_sources = ( await Get( Digest, AddPrefix(prepared_plugin_sources.stripped_source_files.snapshot.digest, "__plugins"), ) if pylint.source_plugins else EMPTY_DIGEST ) pythonpath = list(prepared_python_sources.source_roots) if pylint.source_plugins: # NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must # point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) pythonpath.append("__plugins") input_digest = await Get( Digest, MergeDigests( ( pylint_pex.digest, requirements_pex.digest, config_digest, prefixed_plugin_sources, prepared_python_sources.source_files.snapshot.digest, ) ), ) result = await Get( FallibleProcessResult, PexProcess( pylint_pex, argv=generate_args(source_files=field_set_sources, pylint=pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description=f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return LintResult.from_fallible_process_result( result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)) )
async def bandit_lint_partition(partition: BanditPartition, bandit: Bandit, lint_subsystem: LintSubsystem) -> LintResult: bandit_pex_get = Get( VenvPex, PexRequest( output_filename="bandit.pex", internal_only=True, requirements=PexRequirements(bandit.all_requirements), interpreter_constraints=partition.interpreter_constraints, main=bandit.main, ), ) config_files_get = Get(ConfigFiles, ConfigFilesRequest, bandit.config_request) source_files_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)) bandit_pex, config_files, source_files = await MultiGet( bandit_pex_get, config_files_get, source_files_get) input_digest = await Get( Digest, MergeDigests( (source_files.snapshot.digest, config_files.snapshot.digest))) report_file_name = "bandit_report.txt" if lint_subsystem.reports_dir else None result = await Get( FallibleProcessResult, VenvPexProcess( bandit_pex, argv=generate_argv(source_files, bandit, report_file_name=report_file_name), input_digest=input_digest, description= f"Run Bandit on {pluralize(len(partition.field_sets), 'file')}.", output_files=(report_file_name, ) if report_file_name else None, level=LogLevel.DEBUG, ), ) report = None if report_file_name: report_digest = await Get( Digest, DigestSubset( result.output_digest, PathGlobs( [report_file_name], glob_match_error_behavior=GlobMatchErrorBehavior.warn, description_of_origin="Bandit report file", ), ), ) report = LintReport(report_file_name, report_digest) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)), report=report, )