async def pylint_lint(request: PylintRequest, pylint: Pylint, python_setup: PythonSetup) -> LintResults: if pylint.skip: return LintResults() plugin_targets_request = Get( TransitiveTargets, Addresses( Address.parse(plugin_addr) for plugin_addr in pylint.source_plugins), ) linted_targets_request = Get( Targets, Addresses(field_set.address for field_set in request.field_sets)) plugin_targets, linted_targets = await MultiGet(plugin_targets_request, linted_targets_request) plugin_targets_compatibility_fields = tuple( plugin_tgt[PythonInterpreterCompatibility] for plugin_tgt in plugin_targets.closure if plugin_tgt.has_field(PythonInterpreterCompatibility)) # Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it # doesn't lint those direct dependencies nor does it care about transitive dependencies. per_target_dependencies = await MultiGet( Get(Targets, DependenciesRequest(field_set.dependencies)) for field_set in request.field_sets) # We batch targets by their interpreter constraints to ensure, for example, that all Python 2 # targets run together and all Python 3 targets run together. interpreter_constraints_to_target_setup = defaultdict(set) for field_set, tgt, dependencies in zip(request.field_sets, linted_targets, per_target_dependencies): target_setup = PylintTargetSetup(field_set, Targets([tgt, *dependencies])) interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( ( *(tgt.get(PythonInterpreterCompatibility) for tgt in [tgt, *dependencies]), *plugin_targets_compatibility_fields, ), python_setup, ) or PexInterpreterConstraints(pylint.interpreter_constraints) interpreter_constraints_to_target_setup[interpreter_constraints].add( target_setup) partitions = (PylintPartition( tuple( sorted(target_setups, key=lambda tgt_setup: tgt_setup.field_set.address)), interpreter_constraints, Targets(plugin_targets.closure), ) for interpreter_constraints, target_setups in sorted( interpreter_constraints_to_target_setup.items())) partitioned_results = await MultiGet( Get(LintResult, PylintPartition, partition) for partition in partitions) return LintResults(partitioned_results)
async def bandit_lint( configs: BanditConfigurations, bandit: Bandit, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if bandit.options.skip: return LintResult.noop() # NB: Bandit output depends upon which Python interpreter version it's run with. See # https://github.com/PyCQA/bandit#under-which-version-of-python-should-i-install-bandit. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (config.compatibility for config in configs), python_setup=python_setup ) requirements_pex = await Get[Pex]( PexRequest( output_filename="bandit.pex", requirements=PexRequirements(bandit.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=bandit.get_entry_point(), ) ) config_path: Optional[str] = bandit.options.config config_snapshot = await Get[Snapshot]( PathGlobs( globs=tuple([config_path] if config_path else []), glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--bandit-config`", ) ) all_source_files = await Get[SourceFiles]( AllSourceFilesRequest(config.sources for config in configs) ) specified_source_files = await Get[SourceFiles]( SpecifiedSourceFilesRequest((config.sources, config.origin) for config in configs) ) merged_input_files = await Get[Digest]( MergeDigests( (all_source_files.snapshot.digest, requirements_pex.digest, config_snapshot.digest) ), ) address_references = ", ".join(sorted(config.address.reference() for config in configs)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./bandit.pex", pex_args=generate_args(specified_source_files=specified_source_files, bandit=bandit), input_files=merged_input_files, description=f"Run Bandit on {pluralize(len(configs), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result)
async def flake8_lint( configs: Flake8Configurations, flake8: Flake8, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if flake8.options.skip: return LintResult.noop() # NB: Flake8 output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://flake8.pycqa.org/en/latest/user/invocation.html. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (config.compatibility for config in configs), python_setup) requirements_pex = await Get[Pex](PexRequest( output_filename="flake8.pex", requirements=PexRequirements(flake8.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=flake8.get_entry_point(), )) config_path: Optional[str] = flake8.options.config config_snapshot = await Get[Snapshot](PathGlobs( globs=tuple([config_path] if config_path else []), glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--flake8-config`", )) all_source_files = await Get[SourceFiles](AllSourceFilesRequest( config.sources for config in configs)) specified_source_files = await Get[SourceFiles]( SpecifiedSourceFilesRequest( (config.sources, config.origin) for config in configs)) merged_input_files = await Get[Digest](DirectoriesToMerge(directories=( all_source_files.snapshot.directory_digest, requirements_pex.directory_digest, config_snapshot.directory_digest, )), ) address_references = ", ".join( sorted(config.address.reference() for config in configs)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./flake8.pex", pex_args=generate_args(specified_source_files=specified_source_files, flake8=flake8), input_files=merged_input_files, description= f"Run Flake8 on {pluralize(len(configs), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result)
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: transitive_targets = await Get[TransitiveTargets](Addresses, request.addresses) all_targets = transitive_targets.closure python_targets = [] resource_targets = [] python_requirement_fields = [] for tgt in all_targets: if tgt.has_field(PythonSources): python_targets.append(tgt) if tgt.has_field(PythonRequirementsField): python_requirement_fields.append(tgt[PythonRequirementsField]) # NB: PythonRequirementsFileSources is a subclass of FilesSources. We filter it out so that # requirements.txt is not included in the PEX and so that irrelevant changes to it (e.g. # whitespace changes) do not invalidate the PEX. if tgt.has_field(ResourcesSources) or ( tgt.has_field(FilesSources) and not tgt.has_field(PythonRequirementsFileSources)): resource_targets.append(tgt) interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt.get(PythonInterpreterCompatibility) for tgt in python_targets), python_setup) input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get[ImportablePythonSources]( Targets(python_targets + resource_targets)) input_digests.append(prepared_sources.snapshot.directory_digest) merged_input_digest = await Get[Digest]( DirectoriesToMerge(directories=tuple(input_digests))) requirements = PexRequirements.create_from_requirement_fields( python_requirement_fields, additional_requirements=request.additional_requirements) return PexRequest( output_filename=request.output_filename, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, entry_point=request.entry_point, sources=merged_input_digest, additional_inputs=request.additional_inputs, additional_args=request.additional_args, description=request.description, )
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: transitive_targets = await Get[TransitiveTargets](Addresses, request.addresses) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get[ImportablePythonSources]( Targets(all_targets)) input_digests.append(prepared_sources.snapshot.digest) merged_input_digest = await Get[Digest](MergeDigests(input_digests)) interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility)), python_setup, ) requirements = PexRequirements.create_from_requirement_fields( (tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField)), additional_requirements=request.additional_requirements, ) return PexRequest( output_filename=request.output_filename, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, entry_point=request.entry_point, sources=merged_input_digest, additional_inputs=request.additional_inputs, additional_args=request.additional_args, description=request.description, )
async def setup_pytest_for_target( field_set: PythonTestFieldSet, pytest: PyTest, test_options: TestOptions, python_setup: PythonSetup, ) -> TestTargetSetup: # TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage # configuration via #7490. test_addresses = Addresses((field_set.address, )) transitive_targets = await Get[TransitiveTargets](Addresses, test_addresses) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility)), python_setup, ) # Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints # of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for # CreatePex requests. pex_request = functools.partial( PexRequest, interpreter_constraints=interpreter_constraints) # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly # from zip files. `zipp` has pathologically bad behavior with large zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term, # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata # and then by Pytest). See https://github.com/jaraco/zipp/pull/26. additional_args_for_pytest = ("--not-zip-safe", ) use_coverage = test_options.values.use_coverage plugin_file_digest: Optional[Digest] = (await Get[Digest]( InputFilesContent, COVERAGE_PLUGIN_INPUT) if use_coverage else None) pytest_pex_request = Get[Pex]( PexRequest, pex_request( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), additional_args=additional_args_for_pytest, sources=plugin_file_digest, ), ) requirements_pex_request = Get[Pex](PexFromTargetsRequest( addresses=test_addresses, output_filename="requirements.pex", include_source_files=False, additional_args=additional_args_for_pytest, )) test_runner_pex_request = Get[Pex]( PexRequest, pex_request( output_filename="test_runner.pex", entry_point="pytest:main", interpreter_constraints=interpreter_constraints, additional_args=( "--pex-path", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any pytest transitive requirements will shadow corresponding user # requirements which will lead to problems when APIs that are used by either # `pytest:main` or the tests themselves break between the two versions. ":".join(( pytest_pex_request.subject.output_filename, requirements_pex_request.subject.output_filename, )), ), ), ) prepared_sources_request = Get[ImportablePythonSources]( Targets(all_targets)) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest([(field_set.sources, field_set.origin)], strip_source_roots=True)) requests = ( pytest_pex_request, requirements_pex_request, test_runner_pex_request, prepared_sources_request, specified_source_files_request, ) ( coverage_config, pytest_pex, requirements_pex, test_runner_pex, prepared_sources, specified_source_files, ) = (await MultiGet( Get( CoverageConfig, CoverageConfigRequest( Targets((tgt for tgt in all_targets if tgt.has_field(PythonSources))), is_test_time=True, ), ), *requests, ) if use_coverage else (CoverageConfig(EMPTY_DIGEST), *await MultiGet(*requests))) digests_to_merge = [ coverage_config.digest, prepared_sources.snapshot.digest, requirements_pex.digest, pytest_pex.digest, test_runner_pex.digest, ] input_digest = await Get[Digest](MergeDigests(digests_to_merge)) coverage_args = [] if use_coverage: coverage_args = [ "--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html ] for package in field_set.coverage.determine_packages_to_cover( specified_source_files=specified_source_files): coverage_args.extend(["--cov", package]) specified_source_file_names = sorted(specified_source_files.snapshot.files) return TestTargetSetup( test_runner_pex=test_runner_pex, args=(*pytest.options.args, *coverage_args, *specified_source_file_names), input_digest=input_digest, timeout_seconds=field_set.timeout.calculate_from_global_options( pytest), xml_dir=pytest.options.junit_xml_dir, junit_family=pytest.options.junit_family, )
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: transitive_targets = await Get(TransitiveTargets, Addresses, request.addresses) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets) ) input_digests.append(prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( ( tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility) ), python_setup, ) exact_reqs = PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField) ), additional_requirements=request.additional_requirements, ) requirements = exact_reqs description = request.description if python_setup.requirement_constraints: # In requirement strings Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names exact_req_projects = { canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs } constraints_file_contents = await Get( DigestContents, PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin="the option `--python-setup-requirement-constraints`", ), ) constraints_file_reqs = set( parse_requirements(next(iter(constraints_file_contents)).content.decode()) ) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( f"The constraints file {python_setup.requirement_constraints} does not contain " f"entries for the following requirements: {', '.join(unconstrained_projects)}" ) if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or ( python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES and request.internal_only ): if unconstrained_projects: logger.warning( "Ignoring resolve_all_constraints setting in [python_setup] scope " "because constraints file does not cover all requirements." ) else: requirements = PexRequirements(str(req) for req in constraints_file_reqs) description = description or f"Resolving {python_setup.requirement_constraints}" elif ( python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER and python_setup.resolve_all_constraints_was_set_explicitly() ): raise ValueError( f"[python-setup].resolve_all_constraints is set to " f"{python_setup.resolve_all_constraints.value}, so " f"[python-setup].requirement_constraints must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, entry_point=request.entry_point, sources=merged_input_digest, additional_inputs=request.additional_inputs, additional_args=request.additional_args, description=description, )
async def setup_pytest_for_target( config: PythonTestConfiguration, pytest: PyTest, test_options: TestOptions, python_setup: PythonSetup, ) -> TestTargetSetup: # TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage # configuration via #7490. test_addresses = Addresses((config.address,)) # TODO(John Sirois): PexInterpreterConstraints are gathered in the same way by the # `create_pex_from_target_closure` rule, factor up. transitive_targets = await Get[TransitiveTargets](Addresses, test_addresses) all_targets = transitive_targets.closure # TODO: factor this up? It's mostly duplicated with pex_from_targets.py. python_targets = [] resource_targets = [] for tgt in all_targets: if tgt.has_field(PythonSources): python_targets.append(tgt) # NB: PythonRequirementsFileSources is a subclass of FilesSources. We filter it out so that # requirements.txt is not included in the PEX and so that irrelevant changes to it (e.g. # whitespace changes) do not invalidate the PEX. if tgt.has_field(ResourcesSources) or ( tgt.has_field(FilesSources) and not tgt.has_field(PythonRequirementsFileSources) ): resource_targets.append(tgt) interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt.get(PythonInterpreterCompatibility) for tgt in python_targets), python_setup ) # Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints # of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for # CreatePex requests. pex_request = functools.partial(PexRequest, interpreter_constraints=interpreter_constraints) # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly # from zip files. `zipp` has pathologically bad behavior with large zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term, # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata # and then by Pytest). See https://github.com/jaraco/zipp/pull/26. additional_args_for_pytest = ("--not-zip-safe",) run_coverage = test_options.values.run_coverage plugin_file_digest: Optional[Digest] = ( await Get[Digest](InputFilesContent, COVERAGE_PLUGIN_INPUT) if run_coverage else None ) pytest_pex_request = pex_request( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), additional_args=additional_args_for_pytest, sources=plugin_file_digest, ) requirements_pex_request = LegacyPexFromTargetsRequest( addresses=test_addresses, output_filename="requirements.pex", include_source_files=False, additional_args=additional_args_for_pytest, ) test_runner_pex_request = pex_request( output_filename="test_runner.pex", entry_point="pytest:main", interpreter_constraints=interpreter_constraints, additional_args=( "--pex-path", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any pytest transitive requirements will shadow corresponding user # requirements which will lead to problems when APIs that are used by either # `pytest:main` or the tests themselves break between the two versions. ":".join( (pytest_pex_request.output_filename, requirements_pex_request.output_filename) ), ), ) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. specified_source_files_request = SpecifiedSourceFilesRequest( [(config.sources, config.origin)], strip_source_roots=True ) # TODO(John Sirois): Support exploiting concurrency better: # https://github.com/pantsbuild/pants/issues/9294 # Some awkward code follows in order to execute 5-6 items concurrently given the current state # of MultiGet typing / API. Improve this since we should encourage full concurrency in general. requests: List[Get[Any]] = [ Get[Pex](PexRequest, pytest_pex_request), Get[Pex](LegacyPexFromTargetsRequest, requirements_pex_request), Get[Pex](PexRequest, test_runner_pex_request), Get[ImportablePythonSources](Targets(python_targets + resource_targets)), Get[SourceFiles](SpecifiedSourceFilesRequest, specified_source_files_request), ] if run_coverage: # TODO: update coverage to use the Target API. Also, add tests. hydrated_python_targets = await Get[HydratedTargets]( Addresses(tgt.address for tgt in python_targets) ) requests.append( Get[Coveragerc]( CoveragercRequest(HydratedTargets(hydrated_python_targets), test_time=True) ), ) ( pytest_pex, requirements_pex, test_runner_pex, prepared_sources, specified_source_files, *rest, ) = cast( Union[ Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles], Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles, Coveragerc], ], await MultiGet(requests), ) directories_to_merge = [ prepared_sources.snapshot.directory_digest, requirements_pex.directory_digest, pytest_pex.directory_digest, test_runner_pex.directory_digest, ] if run_coverage: coveragerc = rest[0] directories_to_merge.append(coveragerc.digest) merged_input_files = await Get[Digest]( DirectoriesToMerge(directories=tuple(directories_to_merge)) ) coverage_args = [] if run_coverage: coverage_args = [ "--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html ] for package in config.coverage.determine_packages_to_cover( specified_source_files=specified_source_files ): coverage_args.extend(["--cov", package]) specified_source_file_names = sorted(specified_source_files.snapshot.files) return TestTargetSetup( test_runner_pex=test_runner_pex, args=(*pytest.options.args, *coverage_args, *specified_source_file_names), input_files_digest=merged_input_files, timeout_seconds=config.timeout.calculate_from_global_options(pytest), )
async def pylint_lint( configs: PylintConfigurations, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if pylint.options.skip: return LintResult.noop() # Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it # doesn't lint those direct dependencies nor does it care about transitive dependencies. addresses = [] for config in configs: addresses.append(config.address) addresses.extend(config.dependencies.value or ()) targets = await Get[Targets](Addresses(addresses)) chrooted_python_sources = await Get[ImportablePythonSources](Targets, targets) # NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (config.compatibility for config in configs), python_setup) requirements_pex = await Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements(pylint.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=pylint.get_entry_point(), )) config_path: Optional[str] = pylint.options.config config_snapshot = await Get[Snapshot](PathGlobs( globs=tuple([config_path] if config_path else []), glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) merged_input_files = await Get[Digest](DirectoriesToMerge(directories=( requirements_pex.directory_digest, config_snapshot.directory_digest, chrooted_python_sources.snapshot.directory_digest, )), ) specified_source_files = await Get[SourceFiles]( SpecifiedSourceFilesRequest( ((config.sources, config.origin) for config in configs), strip_source_roots=True)) address_references = ", ".join( sorted(config.address.reference() for config in configs)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./pylint.pex", pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_files=merged_input_files, description= f"Run Pylint on {pluralize(len(configs), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result)
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: transitive_targets = await Get(TransitiveTargets, Addresses, request.addresses) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get(StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets)) input_digests.append( prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility)), python_setup, ) exact_reqs = PexRequirements.create_from_requirement_fields( (tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField)), additional_requirements=request.additional_requirements, ) requirements = exact_reqs if python_setup.requirement_constraints: exact_req_projects = { Requirement.parse(req).project_name for req in exact_reqs } constraints_file_contents = await Get( DigestContents, PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin= "the option `--python-setup-requirement-constraints`", ), ) constraints_file_reqs = set( parse_requirements( next(iter(constraints_file_contents)).content.decode())) constraint_file_projects = { req.project_name for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( f"The constraints file {python_setup.requirement_constraints} does not contain " f"entries for the following requirements: {', '.join(unconstrained_projects)}" ) if python_setup.resolve_all_constraints: if unconstrained_projects: logger.warning( "Ignoring resolve_all_constraints setting in [python_setup] scope" "Because constraints file does not cover all requirements." ) else: requirements = PexRequirements( str(req) for req in constraints_file_reqs) elif python_setup.resolve_all_constraints: raise ValueError( "resolve_all_constraints in the [python-setup] scope is set, so " "requirement_constraints in [python-setup] must also be provided.") return PexRequest( output_filename=request.output_filename, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, entry_point=request.entry_point, sources=merged_input_digest, additional_inputs=request.additional_inputs, additional_args=request.additional_args, description=request.description, )
async def setup_pytest_for_target( field_set: PythonTestFieldSet, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, ) -> TestTargetSetup: test_addresses = Addresses((field_set.address, )) transitive_targets = await Get(TransitiveTargets, Addresses, test_addresses) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility)), python_setup, ) # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly # from zip files. `zipp` has pathologically bad behavior with large zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term, # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata # and then by Pytest). See https://github.com/jaraco/zipp/pull/26. additional_args_for_pytest = ("--not-zip-safe", ) pytest_pex_request = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), interpreter_constraints=interpreter_constraints, additional_args=additional_args_for_pytest, internal_only=True, ), ) # Defaults to zip_safe=False. requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements(test_addresses, internal_only=True), ) test_runner_pex_request = Get( Pex, PexRequest( interpreter_constraints=interpreter_constraints, output_filename="test_runner.pex", entry_point="pytest:main", additional_args=( "--pex-path", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any pytest transitive requirements will shadow corresponding user # requirements which will lead to problems when APIs that are used by either # `pytest:main` or the tests themselves break between the two versions. ":".join(( pytest_pex_request.subject.output_filename, requirements_pex_request.subject.output_filename, )), ), internal_only=True, ), ) prepared_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_request = Get( SourceFiles, SourceFilesRequest([field_set.sources])) ( pytest_pex, requirements_pex, test_runner_pex, prepared_sources, field_set_source_files, ) = await MultiGet( pytest_pex_request, requirements_pex_request, test_runner_pex_request, prepared_sources_request, field_set_source_files_request, ) input_digest = await Get( Digest, MergeDigests(( coverage_config.digest, prepared_sources.source_files.snapshot.digest, requirements_pex.digest, pytest_pex.digest, test_runner_pex.digest, )), ) coverage_args = [] if test_subsystem.use_coverage: cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else ( ".", ) coverage_args = [ "--cov-report=", # Turn off output. *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] return TestTargetSetup( test_runner_pex=test_runner_pex, args=(*pytest.options.args, *coverage_args, *field_set_source_files.files), input_digest=input_digest, source_roots=prepared_sources.source_roots, timeout_seconds=field_set.timeout.calculate_from_global_options( pytest), xml_dir=pytest.options.junit_xml_dir, junit_family=pytest.options.junit_family, execution_slot_variable=pytest.options.execution_slot_var, )
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, global_options: GlobalOptions, ) -> TestSetup: test_addresses = Addresses((request.field_set.address, )) transitive_targets = await Get(TransitiveTargets, Addresses, test_addresses) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility)), python_setup, ) # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly # from zip files. `zipp` has pathologically bad behavior with large zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. Long term, # it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata # and then by Pytest). See https://github.com/jaraco/zipp/pull/26. additional_args_for_pytest = ("--not-zip-safe", ) pytest_pex_request = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), interpreter_constraints=interpreter_constraints, additional_args=additional_args_for_pytest, internal_only=True, ), ) # Defaults to zip_safe=False. requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements(test_addresses, internal_only=True), ) test_runner_pex_request = Get( Pex, PexRequest( interpreter_constraints=interpreter_constraints, output_filename="test_runner.pex", entry_point="pytest:main", additional_args=( "--pex-path", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any pytest transitive requirements will shadow corresponding user # requirements which will lead to problems when APIs that are used by either # `pytest:main` or the tests themselves break between the two versions. ":".join(( pytest_pex_request.subject.output_filename, requirements_pex_request.subject.output_filename, )), ), internal_only=True, ), ) prepared_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_request = Get( SourceFiles, SourceFilesRequest([request.field_set.sources])) ( pytest_pex, requirements_pex, test_runner_pex, prepared_sources, field_set_source_files, ) = await MultiGet( pytest_pex_request, requirements_pex_request, test_runner_pex_request, prepared_sources_request, field_set_source_files_request, ) input_digest = await Get( Digest, MergeDigests(( coverage_config.digest, prepared_sources.source_files.snapshot.digest, requirements_pex.digest, pytest_pex.digest, test_runner_pex.digest, )), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend((f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}")) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: output_files.append(".coverage") cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else ( ".", ) coverage_args = [ "--cov-report=", # Turn off output. *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), } if test_subsystem.force and not request.is_debug: # This is a slightly hacky way to force the process to run: since the env var # value is unique, this input combination will never have been seen before, # and therefore never cached. The two downsides are: # 1. This leaks into the test's environment, albeit with a funky var name that is # unlikely to cause problems in practice. # 2. This run will be cached even though it can never be re-used. # TODO: A more principled way of forcing rules to run? uuid = await Get(UUID, UUIDRequest()) extra_env["__PANTS_FORCE_TEST_RUN__"] = str(uuid) process = await Get( Process, PexProcess( test_runner_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_files=output_files, timeout_seconds=request.field_set.timeout. calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, ), ) return TestSetup(process, results_file_name=results_file_name)
async def pylint_lint( field_sets: PylintFieldSets, pylint: Pylint, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: if pylint.skip: return LintResult.noop() # Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it # doesn't lint those direct dependencies nor does it care about transitive dependencies. addresses_with_dependencies = [] for field_set in field_sets: addresses_with_dependencies.append(field_set.address) addresses_with_dependencies.extend(field_set.dependencies.value or ()) targets = await Get[Targets](Addresses(addresses_with_dependencies)) # NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that # each target runs with its own interpreter constraints. See # http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting. interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (field_set.compatibility for field_set in field_sets), python_setup) # We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies. # Splitting this into two PEXes gives us finer-grained caching. We then merge via `--pex-path`. pylint_pex_request = Get[Pex](PexRequest( output_filename="pylint.pex", requirements=PexRequirements(pylint.get_requirement_specs()), interpreter_constraints=interpreter_constraints, entry_point=pylint.get_entry_point(), )) requirements_pex_request = Get[Pex](PexRequest( output_filename="requirements.pex", requirements=PexRequirements.create_from_requirement_fields( tgt[PythonRequirementsField] for tgt in targets if tgt.has_field(PythonRequirementsField)), interpreter_constraints=interpreter_constraints, )) pylint_runner_pex_request = Get[Pex]( PexRequest( output_filename="pylint_runner.pex", entry_point=pylint.get_entry_point(), interpreter_constraints=interpreter_constraints, additional_args=( "--pex-path", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 # Right now any Pylint transitive requirements will shadow corresponding user # requirements which could lead to problems. ":".join(["pylint.pex", "requirements.pex"]), ), )) config_snapshot_request = Get[Snapshot](PathGlobs( globs=[pylint.config] if pylint.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", )) prepare_python_sources_request = Get[ImportablePythonSources](Targets, targets) specified_source_files_request = Get[SourceFiles]( SpecifiedSourceFilesRequest( ((field_set.sources, field_set.origin) for field_set in field_sets), strip_source_roots=True, )) ( pylint_pex, requirements_pex, pylint_runner_pex, config_snapshot, prepared_python_sources, specified_source_files, ) = cast( Tuple[Pex, Pex, Pex, Snapshot, ImportablePythonSources, SourceFiles], await MultiGet([ pylint_pex_request, requirements_pex_request, pylint_runner_pex_request, config_snapshot_request, prepare_python_sources_request, specified_source_files_request, ]), ) input_digest = await Get[Digest](MergeDigests(( pylint_pex.digest, requirements_pex.digest, pylint_runner_pex.digest, config_snapshot.digest, prepared_python_sources.snapshot.digest, )), ) address_references = ", ".join( sorted(field_set.address.reference() for field_set in field_sets)) process = requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./pylint_runner.pex", pex_args=generate_args(specified_source_files=specified_source_files, pylint=pylint), input_digest=input_digest, description= f"Run Pylint on {pluralize(len(field_sets), 'target')}: {address_references}.", ) result = await Get[FallibleProcessResult](Process, process) return LintResult.from_fallible_process_result(result, linter_name="Pylint")