async def lint( linter: PylintLinter, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if pylint.options.skip: return LintResult.noop() adaptors_with_origins = linter.adaptors_with_origins # Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it # doesn't lint those direct dependencies nor does it care about transitive dependencies. hydrated_targets = [ HydratedTarget(adaptor_with_origin.adaptor) for adaptor_with_origin in adaptors_with_origins ] dependencies = await MultiGet( Get[HydratedTarget](Address, dependency) for dependency in itertools.chain.from_iterable( ht.adaptor.dependencies for ht in hydrated_targets)) chrooted_python_sources = await Get[ImportablePythonSources]( HydratedTargets([*hydrated_targets, *dependencies])) # NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting. interpreter_constraints = PexInterpreterConstraints.create_from_adaptors( (adaptor_with_origin.adaptor for adaptor_with_origin in adaptors_with_origins), python_setup=python_setup, ) requirements_pex = await Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements(pylint.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=pylint.get_entry_point(), )) config_path: Optional[str] = pylint.options.config config_snapshot = await Get[Snapshot](PathGlobs( globs=tuple([config_path] if config_path else []), glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) merged_input_files = await Get[Digest](DirectoriesToMerge(directories=( requirements_pex.directory_digest, config_snapshot.directory_digest, chrooted_python_sources.snapshot.directory_digest, )), ) specified_source_files = await Get[SourceFiles]( LegacySpecifiedSourceFilesRequest(adaptors_with_origins, strip_source_roots=True)) address_references = ", ".join( sorted(adaptor_with_origin.adaptor.address.reference() for adaptor_with_origin in adaptors_with_origins)) request = requirements_pex.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./pylint.pex", pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_files=merged_input_files, description=f"Run Pylint for {address_references}", ) result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest, request) return LintResult.from_fallible_execute_process_result(result)
async def pylint_lint_partition( partition: PylintPartition, pylint: Pylint, first_party_plugins: PylintFirstPartyPlugins ) -> LintResult: requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (field_set.address for field_set in partition.field_sets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition.interpreter_constraints, internal_only=True, direct_deps_only=True, ), ) pylint_pex_get = Get( Pex, PexRequest( output_filename="pylint.pex", internal_only=True, requirements=pylint.pex_requirements( extra_requirements=first_party_plugins.requirement_strings, ), interpreter_constraints=partition.interpreter_constraints, ), ) prepare_python_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies) ) field_set_sources_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets) ) pylint_pex, requirements_pex, prepared_python_sources, field_set_sources = await MultiGet( pylint_pex_get, requirements_pex_get, prepare_python_sources_get, field_set_sources_get, ) pylint_runner_pex, config_files = await MultiGet( Get( VenvPex, PexRequest( output_filename="pylint_runner.pex", interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, pex_path=[pylint_pex, requirements_pex], ), ), Get( ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs) ), ) pythonpath = list(prepared_python_sources.source_roots) if first_party_plugins: pythonpath.append(first_party_plugins.PREFIX) input_digest = await Get( Digest, MergeDigests( ( config_files.snapshot.digest, first_party_plugins.sources_digest, prepared_python_sources.source_files.snapshot.digest, ) ), ) result = await Get( FallibleProcessResult, VenvPexProcess( pylint_runner_pex, argv=generate_argv(field_set_sources, pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description=f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return LintResult.from_fallible_process_result( result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)) )
async def pylint_lint( configs: PylintConfigurations, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if pylint.options.skip: return LintResult.noop() # Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it # doesn't lint those direct dependencies nor does it care about transitive dependencies. addresses = [] for config in configs: addresses.append(config.address) addresses.extend(config.dependencies.value or ()) targets = await Get[Targets](Addresses(addresses)) chrooted_python_sources = await Get[ImportablePythonSources](Targets, targets) # NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (config.compatibility for config in configs), python_setup) requirements_pex = await Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements(pylint.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=pylint.get_entry_point(), )) config_path: Optional[str] = pylint.options.config config_snapshot = await Get[Snapshot](PathGlobs( globs=tuple([config_path] if config_path else []), glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) merged_input_files = await Get[Digest](DirectoriesToMerge(directories=( requirements_pex.directory_digest, config_snapshot.directory_digest, chrooted_python_sources.snapshot.directory_digest, )), ) specified_source_files = await Get[SourceFiles]( SpecifiedSourceFilesRequest( ((config.sources, config.origin) for config in configs), strip_source_roots=True)) address_references = ", ".join( sorted(config.address.reference() for config in configs)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./pylint.pex", pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_files=merged_input_files, description= f"Run Pylint on {pluralize(len(configs), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result)
async def pylint_lint_partition( partition: PylintPartition, pylint: Pylint, first_party_plugins: PylintFirstPartyPlugins) -> LintResult: requirements_pex_get = Get( Pex, RequirementsPexRequest( (t.address for t in partition.root_targets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, ), ) pylint_pex_get = Get( Pex, PexRequest, pylint.to_pex_request( interpreter_constraints=partition.interpreter_constraints, extra_requirements=first_party_plugins.requirement_strings, ), ) prepare_python_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) field_set_sources_get = Get( SourceFiles, SourceFilesRequest(t[PythonSourceField] for t in partition.root_targets)) # Ensure that the empty report dir exists. report_directory_digest_get = Get(Digest, CreateDigest([Directory(REPORT_DIR)])) ( pylint_pex, requirements_pex, prepared_python_sources, field_set_sources, report_directory, ) = await MultiGet( pylint_pex_get, requirements_pex_get, prepare_python_sources_get, field_set_sources_get, report_directory_digest_get, ) pylint_runner_pex, config_files = await MultiGet( Get( VenvPex, VenvPexRequest( PexRequest( output_filename="pylint_runner.pex", interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, pex_path=[pylint_pex, requirements_pex], ), # TODO(John Sirois): Remove this (change to the default of symlinks) when we can # upgrade to a version of Pylint with https://github.com/PyCQA/pylint/issues/1470 # resolved. site_packages_copies=True, ), ), Get(ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs)), ) pythonpath = list(prepared_python_sources.source_roots) if first_party_plugins: pythonpath.append(first_party_plugins.PREFIX) input_digest = await Get( Digest, MergeDigests(( config_files.snapshot.digest, first_party_plugins.sources_digest, prepared_python_sources.source_files.snapshot.digest, report_directory, )), ) result = await Get( FallibleProcessResult, VenvPexProcess( pylint_runner_pex, argv=generate_argv(field_set_sources, pylint), input_digest=input_digest, output_directories=(REPORT_DIR, ), extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, concurrency_available=len(partition.root_targets), description= f"Run Pylint on {pluralize(len(partition.root_targets), 'file')}.", level=LogLevel.DEBUG, ), ) report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR)) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)), report=report, )
async def pylint_lint_partition(partition: PylintPartition, pylint: Pylint) -> LintResult: requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (field_set.address for field_set in partition.field_sets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, direct_deps_only=True, ), ) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in partition.plugin_targets if plugin_tgt.has_field(PythonRequirementsField)) pylint_pex_get = Get( Pex, PexRequest( output_filename="pylint.pex", internal_only=True, requirements=PexRequirements( [*pylint.all_requirements, *plugin_requirements]), interpreter_constraints=partition.interpreter_constraints, ), ) prepare_plugin_sources_get = Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(partition.plugin_targets)) prepare_python_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies)) field_set_sources_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)) ( pylint_pex, requirements_pex, prepared_plugin_sources, prepared_python_sources, field_set_sources, ) = await MultiGet( pylint_pex_get, requirements_pex_get, prepare_plugin_sources_get, prepare_python_sources_get, field_set_sources_get, ) pylint_runner_pex, config_files = await MultiGet( Get( VenvPex, PexRequest( output_filename="pylint_runner.pex", interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, pex_path=[pylint_pex, requirements_pex], ), ), Get(ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs)), ) prefixed_plugin_sources = (await Get( Digest, AddPrefix( prepared_plugin_sources.stripped_source_files.snapshot.digest, "__plugins"), ) if pylint.source_plugins else EMPTY_DIGEST) pythonpath = list(prepared_python_sources.source_roots) if pylint.source_plugins: # NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must # point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) pythonpath.append("__plugins") input_digest = await Get( Digest, MergeDigests(( config_files.snapshot.digest, prefixed_plugin_sources, prepared_python_sources.source_files.snapshot.digest, )), ) result = await Get( FallibleProcessResult, VenvPexProcess( pylint_runner_pex, argv=generate_argv(field_set_sources, pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description= f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)))
async def pylint_lint_partition( partition: PylintPartition, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: # We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies. # Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`. plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in partition.plugin_targets if plugin_tgt.has_field(PythonRequirementsField)) target_requirements = PexRequirements.create_from_requirement_fields( tgt[PythonRequirementsField] for tgt in partition.targets_with_dependencies if tgt.has_field(PythonRequirementsField)) pylint_pex_request = Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements( [*pylint.get_requirement_specs(), *plugin_requirements]), interpreter_constraints=partition.interpreter_constraints, entry_point=pylint.get_entry_point(), )) requirements_pex_request = Get[Pex](PexRequest( output_filename="requirements.pex", requirements=target_requirements, interpreter_constraints=partition.interpreter_constraints, )) # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any Pylint transitive requirements will shadow corresponding user # requirements, which could lead to problems. pylint_runner_pex_args = [ "--pex-path", ":".join(["pylint.pex", "requirements.pex"]) ] if pylint.source_plugins: # NB: See below for why we set PYTHONPATH to load source plugins. This setting is necessary # for PEX to pick up the PYTHONPATH value. pylint_runner_pex_args.append("--inherit-path=fallback") pylint_runner_pex_request = Get[Pex](PexRequest( output_filename="pylint_runner.pex", entry_point=pylint.get_entry_point(), interpreter_constraints=partition.interpreter_constraints, additional_args=pylint_runner_pex_args, )) config_snapshot_request = Get[Snapshot](PathGlobs( globs=[pylint.config] if pylint.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) prepare_plugin_sources_request = Get[ImportablePythonSources]( Targets, partition.plugin_targets) prepare_python_sources_request = Get[ImportablePythonSources]( Targets, partition.targets_with_dependencies) specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest( ((field_set.sources, field_set.origin) for field_set in partition.field_sets), strip_source_roots=True, )) ( pylint_pex, requirements_pex, pylint_runner_pex, config_snapshot, prepared_plugin_sources, prepared_python_sources, specified_source_files, ) = await MultiGet( pylint_pex_request, requirements_pex_request, pylint_runner_pex_request, config_snapshot_request, prepare_plugin_sources_request, prepare_python_sources_request, specified_source_files_request, ) prefixed_plugin_sources = (await Get[Digest](AddPrefix( prepared_plugin_sources.snapshot.digest, "__plugins")) if pylint.source_plugins else EMPTY_DIGEST) input_digest = await Get[Digest](MergeDigests(( pylint_pex.digest, requirements_pex.digest, pylint_runner_pex.digest, config_snapshot.digest, prefixed_plugin_sources, prepared_python_sources.snapshot.digest, )), ) address_references = ", ".join( sorted(field_set.address.reference() for field_set in partition.field_sets)) process = pylint_runner_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./pylint_runner.pex", # NB: Pylint source plugins must be explicitly loaded via PYTHONPATH. The value must # point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) env={"PYTHONPATH": "./__plugins"} if pylint.source_plugins else None, pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_digest=input_digest, description= (f"Run Pylint on {pluralize(len(partition.field_sets), 'target')}: {address_references}." ), ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result, linter_name="Pylint")
async def pylint_lint( field_sets: PylintFieldSets, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if pylint.skip: return LintResult.noop() # Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it # doesn't lint those direct dependencies nor does it care about transitive dependencies. addresses_with_dependencies = [] for field_set in field_sets: addresses_with_dependencies.append(field_set.address) addresses_with_dependencies.extend(field_set.dependencies.value or ()) targets = await Get[Targets](Addresses(addresses_with_dependencies)) # NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (field_set.compatibility for field_set in field_sets), python_setup) # We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies. # Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`. pylint_pex_request = Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements(pylint.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=pylint.get_entry_point(), )) requirements_pex_request = Get[Pex](PexRequest( output_filename="requirements.pex", requirements=PexRequirements.create_from_requirement_fields( tgt[PythonRequirementsField] for tgt in targets if tgt.has_field(PythonRequirementsField)), interpreter_constraints=interpreter_constraints, )) pylint_runner_pex_request = Get[Pex]( PexRequest( output_filename="pylint_runner.pex", entry_point=pylint.get_entry_point(), interpreter_constraints=interpreter_constraints, additional_args=( "--pex-path", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any Pylint transitive requirements will shadow corresponding user # requirements which could lead to problems. ":".join(["pylint.pex", "requirements.pex"]), ), )) config_snapshot_request = Get[Snapshot](PathGlobs( globs=[pylint.config] if pylint.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) prepare_python_sources_request = Get[ImportablePythonSources](Targets, targets) specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest( ((field_set.sources, field_set.origin) for field_set in field_sets), strip_source_roots=True, )) ( pylint_pex, requirements_pex, pylint_runner_pex, config_snapshot, prepared_python_sources, specified_source_files, ) = cast( Tuple[Pex, Pex, Pex, Snapshot, ImportablePythonSources, SourceFiles], await MultiGet([ pylint_pex_request, requirements_pex_request, pylint_runner_pex_request, config_snapshot_request, prepare_python_sources_request, specified_source_files_request, ]), ) input_digest = await Get[Digest](MergeDigests(( pylint_pex.digest, requirements_pex.digest, pylint_runner_pex.digest, config_snapshot.digest, prepared_python_sources.snapshot.digest, )), ) address_references = ", ".join( sorted(field_set.address.reference() for field_set in field_sets)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./pylint_runner.pex", pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_digest=input_digest, description= f"Run Pylint on {pluralize(len(field_sets), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result, linter_name="Pylint")