async def create_python_repl_request(repl: PythonRepl, pex_env: PexEnvironment) -> ReplRequest: requirements_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in repl.targets), internal_only=True), ) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)) requirements_pex, sources = await MultiGet(requirements_request, sources_request) merged_digest = await Get( Digest, MergeDigests( (requirements_pex.digest, sources.source_files.snapshot.digest))) args = pex_env.create_argv(repl.in_chroot(requirements_pex.name), python=requirements_pex.python) chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=requirements_pex.python is not None), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
def get_unstripped_sources( rule_runner: RuleRunner, targets: Iterable[Target], *, include_resources: bool = True, include_files: bool = False, source_roots: list[str] | None = None, extra_args: list[str] | None = None, ) -> PythonSourceFiles: rule_runner.set_options( [ f"--source-root-patterns={source_roots or ['src/python']}", "--no-python-protobuf-infer-runtime-dependency", *(extra_args or []), ], env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) return rule_runner.request( PythonSourceFiles, [ PythonSourceFilesRequest(targets, include_resources=include_resources, include_files=include_files) ], )
async def generate_chroot( request: DistBuildChrootRequest, subsys: SetupPyGeneration ) -> DistBuildChroot: generate_setup = request.exported_target.target.get(GenerateSetupField).value if generate_setup is None: generate_setup = subsys.generate_setup_default if generate_setup: sources = await Get(DistBuildSources, DistBuildChrootRequest, request) generated_setup_py = await Get( GeneratedSetupPy, GenerateSetupPyRequest( request.exported_target, sources, request.interpreter_constraints ), ) # We currently generate a setup.py that expects to be in the source root. # TODO: It might make sense to generate one in the target's directory, for # consistency with the existing setup.py case. working_directory = "" chroot_digest = await Get(Digest, MergeDigests((sources.digest, generated_setup_py.digest))) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.exported_target.target.address]), ) source_files = await Get( PythonSourceFiles, PythonSourceFilesRequest( targets=transitive_targets.closure, include_resources=True, include_files=True ), ) chroot_digest = source_files.source_files.snapshot.digest working_directory = request.exported_target.target.address.spec_path return DistBuildChroot(chroot_digest, working_directory)
async def create_ipython_repl_request(repl: IPythonRepl, ipython: IPython, pex_env: PexEnvironment) -> ReplRequest: # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in ipython_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in repl.targets), internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)) ipython_request = Get( Pex, PexRequest( output_filename="ipython.pex", main=ipython.main, requirements=PexRequirements(ipython.all_requirements), interpreter_constraints=requirements_pex_request. interpreter_constraints, internal_only=True, ), ) requirements_pex, sources, ipython_pex = await MultiGet( requirements_request, sources_request, ipython_request) merged_digest = await Get( Digest, MergeDigests( (requirements_pex.digest, sources.source_files.snapshot.digest, ipython_pex.digest)), ) args = list( pex_env.create_argv(repl.in_chroot(ipython_pex.name), python=ipython_pex.python)) if ipython.options.ignore_cwd: args.append("--ignore-cwd") chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=ipython_pex.python is not None), "PEX_PATH": repl.in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_python_repl_request(request: PythonRepl, pex_env: PexEnvironment, python_setup: PythonSetup) -> ReplRequest: validate_compatible_resolve(request.targets, python_setup) interpreter_constraints, transitive_targets = await MultiGet( Get(InterpreterConstraints, InterpreterConstraintsRequest(request.addresses)), Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)), ) requirements_request = Get(Pex, RequirementsPexRequest(request.addresses)) local_dists_request = Get( LocalDistsPex, LocalDistsPexRequest( request.addresses, internal_only=True, interpreter_constraints=interpreter_constraints, ), ) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) requirements_pex, local_dists, sources = await MultiGet( requirements_request, local_dists_request, sources_request) merged_digest = await Get( Digest, MergeDigests((requirements_pex.digest, local_dists.pex.digest, sources.source_files.snapshot.digest)), ) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(request.in_chroot( requirements_pex.name), python=requirements_pex.python) chrooted_source_roots = [ request.in_chroot(sr) for sr in sources.source_roots ] extra_env = { **complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), "PEX_PATH": request.in_chroot(local_dists.pex.name), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_python_repl_request(repl: PythonRepl, pex_env: PexEnvironment) -> ReplRequest: # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) so # that we can get the interpreter constraints for use in local_dists_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in repl.targets), internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) local_dists_request = Get( LocalDistsPex, LocalDistsPexRequest( Addresses(tgt.address for tgt in repl.targets), internal_only=True, interpreter_constraints=requirements_pex_request. interpreter_constraints, ), ) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)) requirements_pex, local_dists, sources = await MultiGet( requirements_request, local_dists_request, sources_request) merged_digest = await Get( Digest, MergeDigests((requirements_pex.digest, local_dists.pex.digest, sources.source_files.snapshot.digest)), ) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(repl.in_chroot(requirements_pex.name), python=requirements_pex.python) chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), "PEX_PATH": repl.in_chroot(local_dists.pex.name), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def get_sources(request: SetupPySourcesRequest) -> SetupPySources: python_sources_request = PythonSourceFilesRequest(targets=request.targets, include_resources=False, include_files=False) all_sources_request = PythonSourceFilesRequest(targets=request.targets, include_resources=True, include_files=True) python_sources, all_sources = await MultiGet( Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, python_sources_request), Get(StrippedPythonSourceFiles, PythonSourceFilesRequest, all_sources_request), ) python_files = set(python_sources.stripped_source_files.snapshot.files) all_files = set(all_sources.stripped_source_files.snapshot.files) resource_files = all_files - python_files init_py_digest_contents = await Get( DigestContents, DigestSubset(python_sources.stripped_source_files.snapshot.digest, PathGlobs(["**/__init__.py"])), ) packages, namespace_packages, package_data = find_packages( python_files=python_files, resource_files=resource_files, init_py_digest_contents=init_py_digest_contents, py2=request.py2, ) return SetupPySources( digest=all_sources.stripped_source_files.snapshot.digest, packages=packages, namespace_packages=namespace_packages, package_data=package_data, )
async def pylint_first_party_plugins( pylint: Pylint) -> PylintFirstPartyPlugins: if not pylint.source_plugins: return PylintFirstPartyPlugins(FrozenOrderedSet(), FrozenOrderedSet(), EMPTY_DIGEST) plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, pylint.source_plugins) transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses)) requirements_fields: OrderedSet[PythonRequirementsField] = OrderedSet() interpreter_constraints_fields: OrderedSet[ InterpreterConstraintsField] = OrderedSet() for tgt in transitive_targets.closure: if tgt.has_field(PythonRequirementsField): requirements_fields.add(tgt[PythonRequirementsField]) if tgt.has_field(InterpreterConstraintsField): interpreter_constraints_fields.add( tgt[InterpreterConstraintsField]) # NB: Pylint source plugins must be explicitly loaded via PYTHONPATH (i.e. PEX_EXTRA_SYS_PATH). # The value must point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) stripped_sources = await Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure)) prefixed_sources = await Get( Digest, AddPrefix(stripped_sources.stripped_source_files.snapshot.digest, PylintFirstPartyPlugins.PREFIX), ) return PylintFirstPartyPlugins( requirement_strings=PexRequirements.create_from_requirement_fields( requirements_fields, constraints_strings=(), ).req_strings, interpreter_constraints_fields=FrozenOrderedSet( interpreter_constraints_fields), sources_digest=prefixed_sources, )
async def mypy_first_party_plugins(mypy: MyPy) -> MyPyFirstPartyPlugins: if not mypy.source_plugins: return MyPyFirstPartyPlugins(FrozenOrderedSet(), EMPTY_DIGEST, ()) plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, mypy.source_plugins) transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses)) requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in transitive_targets.closure if plugin_tgt.has_field(PythonRequirementsField)) sources = await Get(PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure)) return MyPyFirstPartyPlugins( requirement_strings=requirements.req_strings, sources_digest=sources.source_files.snapshot.digest, source_roots=sources.source_roots, )
def get_unstripped_sources( rule_runner: RuleRunner, targets: Iterable[Target], *, include_resources: bool = True, include_files: bool = False, source_roots: list[str] | None = None, extra_args: list[str] | None = None, ) -> PythonSourceFiles: rule_runner.set_options([ "--backend-packages=pants.backend.python", f"--source-root-patterns={source_roots or ['src/python']}", *(extra_args or []), ]) return rule_runner.request( PythonSourceFiles, [ PythonSourceFilesRequest(targets, include_resources=include_resources, include_files=include_files) ], )
def get_unstripped_sources( self, targets: Iterable[Target], *, include_resources: bool = True, include_files: bool = False, source_roots: Optional[List[str]] = None, extra_args: Optional[List[str]] = None, ) -> PythonSourceFiles: return self.request( PythonSourceFiles, [ PythonSourceFilesRequest(targets, include_resources=include_resources, include_files=include_files), create_options_bootstrapper(args=[ "--backend-packages=pants.backend.python", f"--source-root-patterns={source_roots or ['src/python']}", *(extra_args or []), ]), ], )
async def pylint_lint_partition( partition: PylintPartition, pylint: Pylint, first_party_plugins: PylintFirstPartyPlugins ) -> LintResult: requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (field_set.address for field_set in partition.field_sets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition.interpreter_constraints, internal_only=True, direct_deps_only=True, ), ) pylint_pex_get = Get( Pex, PexRequest( output_filename="pylint.pex", internal_only=True, requirements=pylint.pex_requirements( extra_requirements=first_party_plugins.requirement_strings, ), interpreter_constraints=partition.interpreter_constraints, ), ) prepare_python_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies) ) field_set_sources_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets) ) pylint_pex, requirements_pex, prepared_python_sources, field_set_sources = await MultiGet( pylint_pex_get, requirements_pex_get, prepare_python_sources_get, field_set_sources_get, ) pylint_runner_pex, config_files = await MultiGet( Get( VenvPex, PexRequest( output_filename="pylint_runner.pex", interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, pex_path=[pylint_pex, requirements_pex], ), ), Get( ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs) ), ) pythonpath = list(prepared_python_sources.source_roots) if first_party_plugins: pythonpath.append(first_party_plugins.PREFIX) input_digest = await Get( Digest, MergeDigests( ( config_files.snapshot.digest, first_party_plugins.sources_digest, prepared_python_sources.source_files.snapshot.digest, ) ), ) result = await Get( FallibleProcessResult, VenvPexProcess( pylint_runner_pex, argv=generate_argv(field_set_sources, pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description=f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return LintResult.from_fallible_process_result( result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)) )
async def create_python_binary_run_request( field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults, pex_env: PexEnvironment, ) -> RunRequest: entry_point = field_set.entry_point.value if entry_point is None: binary_source_paths = await Get( Paths, PathGlobs, field_set.sources.path_globs(FilesNotFoundBehavior.error)) if len(binary_source_paths.files) != 1: raise InvalidFieldException( "No `entry_point` was set for the target " f"{repr(field_set.address)}, so it must have exactly one source, but it has " f"{len(binary_source_paths.files)}") entry_point_path = binary_source_paths.files[0] source_root = await Get( SourceRoot, SourceRootRequest, SourceRootRequest.for_file(entry_point_path), ) entry_point = PythonBinarySources.translate_source_file_to_entry_point( os.path.relpath(entry_point_path, source_root.path)) transitive_targets = await Get(TransitiveTargets, Addresses([field_set.address])) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in runner_pex_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([field_set.address], internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) output_filename = f"{field_set.address.target_name}.pex" runner_pex_request = Get( Pex, PexRequest( output_filename=output_filename, interpreter_constraints=requirements_pex_request. interpreter_constraints, additional_args=field_set.generate_additional_args( python_binary_defaults), internal_only=True, # Note that the entry point file is not in the Pex itself, but on the # PEX_PATH. This works fine! entry_point=entry_point, ), ) requirements, sources, runner_pex = await MultiGet(requirements_request, sources_request, runner_pex_request) merged_digest = await Get( Digest, MergeDigests([ requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) args = pex_env.create_argv(in_chroot(runner_pex.name), python=runner_pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=runner_pex.python is not None), "PEX_PATH": in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets ) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses) ) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets) ) input_digests.append(prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup ) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or PexInterpreterConstraints( python_setup.interpreter_constraints ) exact_reqs = PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField) ), additional_requirements=request.additional_requirements, ) requirements = exact_reqs description = request.description if python_setup.requirement_constraints: # In requirement strings Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names exact_req_projects = { canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs } constraints_file_contents = await Get( DigestContents, PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin="the option `--python-setup-requirement-constraints`", ), ) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=python_setup.requirement_constraints, ) ) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( f"The constraints file {python_setup.requirement_constraints} does not contain " f"entries for the following requirements: {', '.join(unconstrained_projects)}" ) if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or ( python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES and request.internal_only ): if unconstrained_projects: logger.warning( "Ignoring resolve_all_constraints setting in [python_setup] scope " "because constraints file does not cover all requirements." ) else: requirements = PexRequirements(str(req) for req in constraints_file_reqs) description = description or f"Resolving {python_setup.requirement_constraints}" elif ( python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER and python_setup.resolve_all_constraints_was_set_explicitly() ): raise ValueError( f"[python-setup].resolve_all_constraints is set to " f"{python_setup.resolve_all_constraints.value}, so " f"[python-setup].requirement_constraints must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, entry_point=request.entry_point, sources=merged_input_digest, additional_inputs=request.additional_inputs, additional_args=request.additional_args, description=description, )
async def create_pex_binary_run_request(field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment) -> RunRequest: run_in_sandbox = field_set.run_in_sandbox.value entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) addresses = [field_set.address] interpreter_constraints = await Get( InterpreterConstraints, InterpreterConstraintsRequest(addresses)) pex_filename = (field_set.address.generated_name.replace(".", "_") if field_set.address.generated_name else field_set.address.target_name) pex_get = Get( Pex, PexFromTargetsRequest( [field_set.address], output_filename=f"{pex_filename}.pex", internal_only=True, include_source_files=False, # Note that the file for first-party entry points is not in the PEX itself. In that # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`. main=entry_point.val or field_set.script.value, additional_args=( *field_set.generate_additional_args(pex_binary_defaults), # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH # below, it's important for any app that re-executes itself that these environment # variables are not stripped. "--no-strip-pex-env", ), ), ) sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) pex, sources = await MultiGet(pex_get, sources_get) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [field_set.address], internal_only=True, interpreter_constraints=interpreter_constraints, sources=sources, ), ) input_digests = [ pex.digest, local_dists.pex.digest, # Note regarding inline mode: You might think that the sources don't need to be copied # into the chroot when using inline sources. But they do, because some of them might be # codegenned, and those won't exist in the inline source tree. Rather than incurring the # complexity of figuring out here which sources were codegenned, we copy everything. # The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline # sources will take precedence and their copies in the chroot will be ignored. local_dists.remaining_sources.source_files.snapshot.digest, ] merged_digest = await Get(Digest, MergeDigests(input_digests)) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] # The order here is important: we want the in-repo sources to take precedence over their # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode). source_roots = [ *([] if run_in_sandbox else sources.source_roots), *chrooted_source_roots, ] extra_env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), "PEX_PATH": in_chroot(local_dists.pex.name), "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def collect_fixture_configs( _request: CollectFixtureConfigsRequest, pytest: PyTest, python_setup: PythonSetup, test_extra_env: TestExtraEnv, targets: Targets, ) -> CollectedJVMLockfileFixtureConfigs: addresses = [tgt.address for tgt in targets] transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(addresses)) all_targets = transitive_targets.closure interpreter_constraints = InterpreterConstraints.create_from_targets( all_targets, python_setup) pytest_pex, requirements_pex, prepared_sources, root_sources = await MultiGet( Get( Pex, PexRequest( output_filename="pytest.pex", requirements=pytest.pex_requirements(), interpreter_constraints=interpreter_constraints, internal_only=True, ), ), Get(Pex, RequirementsPexRequest(addresses)), Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True, include_resources=True), ), Get( PythonSourceFiles, PythonSourceFilesRequest(targets), ), ) script_content = FileContent(path="collect-fixtures.py", content=COLLECTION_SCRIPT.encode(), is_executable=True) script_digest = await Get(Digest, CreateDigest([script_content])) pytest_runner_pex_get = Get( VenvPex, PexRequest( output_filename="pytest_runner.pex", interpreter_constraints=interpreter_constraints, main=EntryPoint(PurePath(script_content.path).stem), sources=script_digest, internal_only=True, pex_path=[ pytest_pex, requirements_pex, ], ), ) config_file_dirs = list( group_by_dir(prepared_sources.source_files.files).keys()) config_files_get = Get( ConfigFiles, ConfigFilesRequest, pytest.config_request(config_file_dirs), ) pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get, config_files_get) pytest_config_digest = config_files.snapshot.digest input_digest = await Get( Digest, MergeDigests(( prepared_sources.source_files.snapshot.digest, pytest_config_digest, )), ) extra_env = { "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), **test_extra_env.env, } process = await Get( Process, VenvPexProcess( pytest_runner_pex, argv=[ name for name in root_sources.source_files.files if name.endswith(".py") ], extra_env=extra_env, input_digest=input_digest, output_files=("tests.json", ), description="Collect test lockfile requirements from all tests.", level=LogLevel.DEBUG, cache_scope=ProcessCacheScope.PER_SESSION, ), ) result = await Get(ProcessResult, Process, process) digest_contents = await Get(DigestContents, Digest, result.output_digest) assert len(digest_contents) == 1 assert digest_contents[0].path == "tests.json" raw_config_data = json.loads(digest_contents[0].content) configs = [] for item in raw_config_data: config = JVMLockfileFixtureConfig( definition=JVMLockfileFixtureDefinition.from_kwargs( item["kwargs"]), test_file_path=item["test_file_path"], ) configs.append(config) return CollectedJVMLockfileFixtureConfigs(configs)
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.field_set.address]) ) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup ) requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([request.field_set.address], internal_only=True), ) pytest_pex_request = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), interpreter_constraints=interpreter_constraints, internal_only=True, ), ) prepared_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True) ) # Create any assets that the test depends on through the `runtime_package_dependencies` field. assets: Tuple[BuiltPackage, ...] = () unparsed_runtime_packages = ( request.field_set.runtime_package_dependencies.to_unparsed_address_inputs() ) if unparsed_runtime_packages.values: runtime_package_targets = await Get( Targets, UnparsedAddressInputs, unparsed_runtime_packages ) field_sets_per_target = await Get( FieldSetsPerTarget, FieldSetsPerTargetRequest(PackageFieldSet, runtime_package_targets), ) assets = await MultiGet( Get(BuiltPackage, PackageFieldSet, field_set) for field_set in field_sets_per_target.field_sets ) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_request = Get( SourceFiles, SourceFilesRequest([request.field_set.sources]) ) pytest_pex, requirements_pex, prepared_sources, field_set_source_files = await MultiGet( pytest_pex_request, requirements_pex_request, prepared_sources_request, field_set_source_files_request, ) pytest_runner_pex = await Get( VenvPex, PexRequest( output_filename="pytest_runner.pex", interpreter_constraints=interpreter_constraints, # TODO(John Sirois): Switch to ConsoleScript once Pex supports discovering console # scripts via the PEX_PATH: https://github.com/pantsbuild/pex/issues/1257 main=EntryPoint("pytest"), internal_only=True, pex_path=[pytest_pex, requirements_pex], ), ) input_digest = await Get( Digest, MergeDigests( ( coverage_config.digest, prepared_sources.source_files.snapshot.digest, *(binary.digest for binary in assets), ) ), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend( (f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}") ) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: output_files.append(".coverage") cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else (".",) coverage_args = [ "--cov-report=", # Turn off output. *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), } extra_env.update(test_extra_env.env) # Cache test runs only if they are successful, or not at all if `--test-force`. cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL process = await Get( Process, VenvPexProcess( pytest_runner_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_files=output_files, timeout_seconds=request.field_set.timeout.calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, cache_scope=cache_scope, ), ) return TestSetup(process, results_file_name=results_file_name)
async def pylint_lint_partition( partition: PylintPartition, pylint: Pylint, first_party_plugins: PylintFirstPartyPlugins) -> LintResult: requirements_pex_get = Get( Pex, RequirementsPexRequest( (t.address for t in partition.root_targets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, ), ) pylint_pex_get = Get( Pex, PexRequest, pylint.to_pex_request( interpreter_constraints=partition.interpreter_constraints, extra_requirements=first_party_plugins.requirement_strings, ), ) prepare_python_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) field_set_sources_get = Get( SourceFiles, SourceFilesRequest(t[PythonSourceField] for t in partition.root_targets)) # Ensure that the empty report dir exists. report_directory_digest_get = Get(Digest, CreateDigest([Directory(REPORT_DIR)])) ( pylint_pex, requirements_pex, prepared_python_sources, field_set_sources, report_directory, ) = await MultiGet( pylint_pex_get, requirements_pex_get, prepare_python_sources_get, field_set_sources_get, report_directory_digest_get, ) pylint_runner_pex, config_files = await MultiGet( Get( VenvPex, VenvPexRequest( PexRequest( output_filename="pylint_runner.pex", interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, pex_path=[pylint_pex, requirements_pex], ), # TODO(John Sirois): Remove this (change to the default of symlinks) when we can # upgrade to a version of Pylint with https://github.com/PyCQA/pylint/issues/1470 # resolved. site_packages_copies=True, ), ), Get(ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs)), ) pythonpath = list(prepared_python_sources.source_roots) if first_party_plugins: pythonpath.append(first_party_plugins.PREFIX) input_digest = await Get( Digest, MergeDigests(( config_files.snapshot.digest, first_party_plugins.sources_digest, prepared_python_sources.source_files.snapshot.digest, report_directory, )), ) result = await Get( FallibleProcessResult, VenvPexProcess( pylint_runner_pex, argv=generate_argv(field_set_sources, pylint), input_digest=input_digest, output_directories=(REPORT_DIR, ), extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, concurrency_available=len(partition.root_targets), description= f"Run Pylint on {pluralize(len(partition.root_targets), 'file')}.", level=LogLevel.DEBUG, ), ) report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR)) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)), report=report, )
async def create_pex_binary_run_request( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment, ) -> RunRequest: entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in runner_pex_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([field_set.address], internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) output_filename = f"{field_set.address.target_name}.pex" runner_pex_request = Get( Pex, PexRequest( output_filename=output_filename, interpreter_constraints=requirements_pex_request. interpreter_constraints, additional_args=field_set.generate_additional_args( pex_binary_defaults), internal_only=True, # Note that the entry point file is not in the PEX itself. It's loaded by setting # `PEX_EXTRA_SYS_PATH`. # TODO(John Sirois): Support ConsoleScript in PexBinary targets: # https://github.com/pantsbuild/pants/issues/11619 main=entry_point.val, ), ) requirements, sources, runner_pex = await MultiGet(requirements_request, sources_request, runner_pex_request) merged_digest = await Get( Digest, MergeDigests([ requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) args = pex_env.create_argv(in_chroot(runner_pex.name), python=runner_pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=runner_pex.python is not None), "PEX_PATH": in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, complete_env: CompleteEnvironment, ) -> TestSetup: transitive_targets, plugin_setups = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])), Get(AllPytestPluginSetups, AllPytestPluginSetupsRequest(request.field_set.address)), ) all_targets = transitive_targets.closure interpreter_constraints = InterpreterConstraints.create_from_targets( all_targets, python_setup) requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( [request.field_set.address], internal_only=True, resolve_and_lockfile=request.field_set.resolve. resolve_and_lockfile(python_setup), ), ) pytest_pex_get = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=pytest.pex_requirements(), interpreter_constraints=interpreter_constraints, internal_only=True, ), ) # Ensure that the empty extra output dir exists. extra_output_directory_digest_get = Get( Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)])) prepared_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_get = Get( SourceFiles, SourceFilesRequest([request.field_set.sources])) ( pytest_pex, requirements_pex, prepared_sources, field_set_source_files, extra_output_directory_digest, ) = await MultiGet( pytest_pex_get, requirements_pex_get, prepared_sources_get, field_set_source_files_get, extra_output_directory_digest_get, ) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [request.field_set.address], interpreter_constraints=interpreter_constraints, sources=prepared_sources, ), ) pytest_runner_pex_get = Get( VenvPex, PexRequest( output_filename="pytest_runner.pex", interpreter_constraints=interpreter_constraints, main=pytest.main, internal_only=True, pex_path=[pytest_pex, requirements_pex, local_dists.pex], ), ) config_files_get = Get( ConfigFiles, ConfigFilesRequest, pytest.config_request(field_set_source_files.snapshot.dirs), ) pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get, config_files_get) input_digest = await Get( Digest, MergeDigests(( coverage_config.digest, local_dists.remaining_sources.source_files.snapshot.digest, config_files.snapshot.digest, extra_output_directory_digest, *(plugin_setup.digest for plugin_setup in plugin_setups), )), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend((f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}")) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: pytest.validate_pytest_cov_included() output_files.append(".coverage") if coverage_subsystem.filter: cov_args = [f"--cov={morf}" for morf in coverage_subsystem.filter] else: # N.B.: Passing `--cov=` or `--cov=.` to communicate "record coverage for all sources" # fails in certain contexts as detailed in: # https://github.com/pantsbuild/pants/issues/12390 # Instead we focus coverage on just the directories containing python source files # materialized to the Process chroot. cov_args = [ f"--cov={source_root}" for source_root in prepared_sources.source_roots ] coverage_args = [ "--cov-report=", # Turn off output. f"--cov-config={coverage_config.path}", *cov_args, ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), **test_extra_env.env, # NOTE: `complete_env` intentionally after `test_extra_env` to allow overriding within # `python_tests` **complete_env.get_subset(request.field_set.extra_env_vars.value or ()), } # Cache test runs only if they are successful, or not at all if `--test-force`. cache_scope = (ProcessCacheScope.PER_SESSION if test_subsystem.force else ProcessCacheScope.SUCCESSFUL) process = await Get( Process, VenvPexProcess( pytest_runner_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_directories=(_EXTRA_OUTPUT_DIR, ), output_files=output_files, timeout_seconds=request.field_set.timeout. calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, cache_scope=cache_scope, ), ) return TestSetup(process, results_file_name=results_file_name)
async def pex_from_targets( request: PexFromTargetsRequest, python_setup: PythonSetup, constraints_file: MaybeConstraintsFile, ) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses)) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get(StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets)) input_digests.append( prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or PexInterpreterConstraints( python_setup.interpreter_constraints) exact_reqs = PexRequirements.create_from_requirement_fields( (tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField)), additional_requirements=request.additional_requirements, ) requirements = exact_reqs repository_pex: Pex | None = None description = request.description if constraints_file.path: constraints_file_contents = await Get(DigestContents, Digest, constraints_file.digest) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=constraints_file.path, )) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names exact_req_projects = { canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs } constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: constraints_descr = ( f"constraints file {constraints_file.path}" if python_setup.requirement_constraints else f"_python_constraints target {python_setup.requirement_constraints_target}" ) logger.warning( f"The {constraints_descr} does not contain entries for the following " f"requirements: {', '.join(unconstrained_projects)}") if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or ( python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES and request.internal_only): if unconstrained_projects: logger.warning( "Ignoring `[python_setup].resolve_all_constraints` option because constraints " "file does not cover all requirements.") else: repository_pex = await Get( Pex, PexRequest( description= f"Resolving {python_setup.requirement_constraints}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements( str(req) for req in constraints_file_reqs), interpreter_constraints=interpreter_constraints, platforms=request.platforms, ), ) elif (python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER and python_setup.resolve_all_constraints_was_set_explicitly()): raise ValueError( "[python-setup].resolve_all_constraints is set to " f"{python_setup.resolve_all_constraints.value}, so " "either [python-setup].requirement_constraints or " "[python-setup].requirement_constraints_target must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, main=request.main, sources=merged_input_digest, additional_inputs=request.additional_inputs, repository_pex=repository_pex, additional_args=request.additional_args, description=description, )
async def generate_coverage_reports( merged_coverage_data: MergedCoverageData, coverage_setup: CoverageSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, global_options: GlobalOptions, ) -> CoverageReports: """Takes all Python test results and generates a single coverage report.""" transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(merged_coverage_data.addresses)) sources = await Get( PythonSourceFiles, # Coverage sometimes includes non-Python files in its `.coverage` data. We need to # ensure that they're present when generating the report. We include all the files included # by `pytest_runner.py`. PythonSourceFilesRequest(transitive_targets.closure, include_files=True, include_resources=True), ) input_digest = await Get( Digest, MergeDigests(( merged_coverage_data.coverage_data, coverage_config.digest, sources.source_files.snapshot.digest, )), ) pex_processes = [] report_types = [] result_snapshot = await Get(Snapshot, Digest, merged_coverage_data.coverage_data) coverage_reports: list[CoverageReport] = [] for report_type in coverage_subsystem.reports: if report_type == CoverageReportType.RAW: coverage_reports.append( FilesystemCoverageReport( # We don't know yet if the coverage is sufficient, so we let some other report # trigger the failure if necessary. coverage_insufficient=False, report_type=CoverageReportType.RAW.value, result_snapshot=result_snapshot, directory_to_materialize_to=coverage_subsystem.output_dir, report_file=coverage_subsystem.output_dir / ".coverage", )) continue report_types.append(report_type) output_file = (f"coverage.{report_type.value}" if report_type in { CoverageReportType.XML, CoverageReportType.JSON } else None) args = [report_type.report_name, f"--rcfile={coverage_config.path}"] if coverage_subsystem.fail_under is not None: args.append(f"--fail-under={coverage_subsystem.fail_under}") pex_processes.append( VenvPexProcess( coverage_setup.pex, argv=tuple(args), input_digest=input_digest, output_directories=("htmlcov", ) if report_type == CoverageReportType.HTML else None, output_files=(output_file, ) if output_file else None, description= f"Generate Pytest {report_type.report_name} coverage report.", level=LogLevel.DEBUG, )) results = await MultiGet( Get(FallibleProcessResult, VenvPexProcess, process) for process in pex_processes) for proc, res in zip(pex_processes, results): if res.exit_code not in {0, 2}: # coverage.py uses exit code 2 if --fail-under triggers, in which case the # reports are still generated. raise ProcessExecutionFailure( res.exit_code, res.stdout, res.stderr, proc.description, local_cleanup=global_options.options. process_execution_local_cleanup, ) # In practice if one result triggers --fail-under, they all will, but no need to rely on that. result_exit_codes = tuple(res.exit_code for res in results) result_stdouts = tuple(res.stdout for res in results) result_snapshots = await MultiGet( Get(Snapshot, Digest, res.output_digest) for res in results) coverage_reports.extend( _get_coverage_report(coverage_subsystem.output_dir, report_type, exit_code != 0, stdout, snapshot) for (report_type, exit_code, stdout, snapshot) in zip( report_types, result_exit_codes, result_stdouts, result_snapshots)) return CoverageReports(tuple(coverage_reports))
async def mypy_typecheck_partition(partition: MyPyPartition, mypy: MyPy) -> TypecheckResult: plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, mypy.source_plugins) plugin_transitive_targets_request = Get( TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses)) plugin_transitive_targets, launcher_script = await MultiGet( plugin_transitive_targets_request, Get(Digest, CreateDigest([LAUNCHER_FILE]))) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in plugin_transitive_targets.closure if plugin_tgt.has_field(PythonRequirementsField)) # If the user did not set `--python-version` already, we set it ourselves based on their code's # interpreter constraints. This determines what AST is used by MyPy. python_version = ( None if partition.python_version_already_configured else partition.interpreter_constraints.minimum_python_version()) # MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6, # and 3.7. However, typed-ast does not understand 3.8, so instead we must run MyPy with # Python 3.8 when relevant. We only do this if <3.8 can't be used, as we don't want a # loose requirement like `>=3.6` to result in requiring Python 3.8, which would error if # 3.8 is not installed on the machine. tool_interpreter_constraints = PexInterpreterConstraints(( "CPython>=3.8", ) if ( mypy.options.is_default("interpreter_constraints") and partition.interpreter_constraints.requires_python38_or_newer() ) else mypy.interpreter_constraints) plugin_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(plugin_transitive_targets.closure)) typechecked_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) # Normally, this `requirements.pex` would be merged with mypy.pex via `--pex-path`. However, # this will cause a runtime error if the interpreter constraints are different between the # PEXes and they have incompatible wheels. # # Instead, we teach MyPy about the requirements by extracting the distributions from # requirements.pex and setting EXTRACTED_WHEELS, which our custom launcher script then # looks for. # # Conventionally, MyPy users might instead set `MYPYPATH` for this. However, doing this # results in type checking the requirements themselves. requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (addr for addr in partition.field_set_addresses), hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, ), ) mypy_pex_request = Get( Pex, PexRequest( output_filename="mypy.pex", internal_only=True, sources=launcher_script, requirements=PexRequirements( itertools.chain(mypy.all_requirements, plugin_requirements)), interpreter_constraints=tool_interpreter_constraints, entry_point=PurePath(LAUNCHER_FILE.path).stem, ), ) config_digest_request = Get(Digest, PathGlobs, config_path_globs(mypy)) ( plugin_sources, typechecked_sources, mypy_pex, requirements_pex, config_digest, ) = await MultiGet( plugin_sources_request, typechecked_sources_request, mypy_pex_request, requirements_pex_request, config_digest_request, ) typechecked_srcs_snapshot = typechecked_sources.source_files.snapshot file_list_path = "__files.txt" python_files = "\n".join( determine_python_files( typechecked_sources.source_files.snapshot.files)) create_file_list_request = Get( Digest, CreateDigest([FileContent(file_list_path, python_files.encode())]), ) file_list_digest, extracted_pex_distributions = await MultiGet( create_file_list_request, Get(ExtractedPexDistributions, Pex, requirements_pex)) merged_input_files = await Get( Digest, MergeDigests([ file_list_digest, plugin_sources.source_files.snapshot.digest, typechecked_srcs_snapshot.digest, mypy_pex.digest, extracted_pex_distributions.digest, config_digest, ]), ) all_used_source_roots = sorted( set( itertools.chain(plugin_sources.source_roots, typechecked_sources.source_roots))) env = { "PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots), "EXTRACTED_WHEELS": ":".join(extracted_pex_distributions.wheel_directory_paths), } result = await Get( FallibleProcessResult, PexProcess( mypy_pex, argv=generate_argv(mypy, file_list_path=file_list_path, python_version=python_version), input_digest=merged_input_files, extra_env=env, description= f"Run MyPy on {pluralize(len(typechecked_srcs_snapshot.files), 'file')}.", level=LogLevel.DEBUG, ), ) return TypecheckResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)))
async def generate_coverage_reports( merged_coverage_data: MergedCoverageData, coverage_setup: CoverageSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, all_used_addresses: Addresses, ) -> CoverageReports: """Takes all Python test results and generates a single coverage report.""" transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(all_used_addresses)) sources = await Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_resources=False), ) input_digest = await Get( Digest, MergeDigests( ( merged_coverage_data.coverage_data, coverage_config.digest, coverage_setup.pex.digest, sources.source_files.snapshot.digest, ) ), ) pex_processes = [] report_types = [] result_snapshot = await Get(Snapshot, Digest, merged_coverage_data.coverage_data) coverage_reports: List[CoverageReport] = [] for report_type in coverage_subsystem.reports: if report_type == CoverageReportType.RAW: coverage_reports.append( FilesystemCoverageReport( report_type=CoverageReportType.RAW.value, result_snapshot=result_snapshot, directory_to_materialize_to=coverage_subsystem.output_dir, report_file=coverage_subsystem.output_dir / ".coverage", ) ) continue report_types.append(report_type) output_file = ( f"coverage.{report_type.value}" if report_type in {CoverageReportType.XML, CoverageReportType.JSON} else None ) pex_processes.append( PexProcess( coverage_setup.pex, # We pass `--ignore-errors` because Pants dynamically injects missing `__init__.py` # files and this will cause Coverage to fail. argv=(report_type.report_name, "--ignore-errors"), input_digest=input_digest, output_directories=("htmlcov",) if report_type == CoverageReportType.HTML else None, output_files=(output_file,) if output_file else None, description=f"Generate Pytest {report_type.report_name} coverage report.", level=LogLevel.DEBUG, ) ) results = await MultiGet(Get(ProcessResult, PexProcess, process) for process in pex_processes) result_stdouts = tuple(res.stdout for res in results) result_snapshots = await MultiGet(Get(Snapshot, Digest, res.output_digest) for res in results) coverage_reports.extend( _get_coverage_report(coverage_subsystem.output_dir, report_type, stdout, snapshot) for (report_type, stdout, snapshot) in zip(report_types, result_stdouts, result_snapshots) ) return CoverageReports(tuple(coverage_reports))
async def pex_from_targets( request: PexFromTargetsRequest, python_setup: PythonSetup, constraints_file: MaybeConstraintsFile, ) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets ) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses) ) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets) ) input_digests.append(prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup ) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or PexInterpreterConstraints( python_setup.interpreter_constraints ) exact_reqs = PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField) ), additional_requirements=request.additional_requirements, ) requirements = exact_reqs repository_pex: Pex | None = None description = request.description if constraints_file.path: constraints_file_contents = await Get(DigestContents, Digest, constraints_file.digest) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=constraints_file.path, ) ) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names url_reqs = set() # E.g., 'foobar@ git+https://github.com/foo/bar.git@branch' name_reqs = set() # E.g., foobar>=1.2.3 name_req_projects = set() for req_str in exact_reqs: req = Requirement.parse(req_str) if req.url: # type: ignore[attr-defined] url_reqs.add(req) else: name_reqs.add(req) name_req_projects.add(canonicalize_project_name(req.project_name)) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } # Constraints files must only contain name reqs, not URL reqs (those are already # constrained by their very nature). See https://github.com/pypa/pip/issues/8210. unconstrained_projects = name_req_projects - constraint_file_projects if unconstrained_projects: constraints_descr = ( f"constraints file {constraints_file.path}" if python_setup.requirement_constraints else f"_python_constraints target {python_setup.requirement_constraints_target}" ) logger.warning( f"The {constraints_descr} does not contain entries for the following " f"requirements: {', '.join(unconstrained_projects)}" ) if python_setup.resolve_all_constraints: if unconstrained_projects: logger.warning( "Ignoring `[python_setup].resolve_all_constraints` option because constraints " "file does not cover all requirements." ) else: # To get a full set of requirements we must add the URL requirements to the # constraints file, since the latter cannot contain URL requirements. # NB: We can only add the URL requirements we know about here, i.e., those that # are transitive deps of the targets in play. There may be others in the repo. # So we may end up creating a few different repository pexes, each with identical # name requirements but different subsets of URL requirements. Fortunately since # all these repository pexes will have identical pinned versions of everything, # this is not a correctness issue, only a performance one. # TODO: Address this as part of providing proper lockfile support. However we # generate lockfiles, they must be able to include URL requirements. all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)} repository_pex = await Get( Pex, PexRequest( description=f"Resolving {python_setup.requirement_constraints}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements(all_constraints), interpreter_constraints=interpreter_constraints, platforms=request.platforms, additional_args=["-vvv"], ), ) elif ( python_setup.resolve_all_constraints and python_setup.resolve_all_constraints_was_set_explicitly() ): raise ValueError( "[python-setup].resolve_all_constraints is enabled, so either " "[python-setup].requirement_constraints or " "[python-setup].requirement_constraints_target must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, main=request.main, sources=merged_input_digest, additional_inputs=request.additional_inputs, repository_pex=repository_pex, additional_args=request.additional_args, description=description, )
async def mypy_typecheck_partition(partition: MyPyPartition, mypy: MyPy) -> TypecheckResult: plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, mypy.source_plugins) plugin_transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses) ) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in plugin_transitive_targets.closure if plugin_tgt.has_field(PythonRequirementsField) ) # If the user did not set `--python-version` already, we set it ourselves based on their code's # interpreter constraints. This determines what AST is used by MyPy. python_version = ( None if partition.python_version_already_configured else partition.interpreter_constraints.minimum_python_version() ) # MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6, # and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with # Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a # loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if # 3.8+ is not installed on the machine. tool_interpreter_constraints = ( partition.interpreter_constraints if ( mypy.options.is_default("interpreter_constraints") and partition.interpreter_constraints.requires_python38_or_newer() ) else PexInterpreterConstraints(mypy.interpreter_constraints) ) plugin_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(plugin_transitive_targets.closure) ) closure_sources_get = Get(PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) roots_sources_get = Get( SourceFiles, SourceFilesRequest(tgt.get(PythonSources) for tgt in partition.root_targets) ) requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in partition.root_targets), hardcoded_interpreter_constraints=partition.interpreter_constraints, internal_only=True, ), ) # TODO(John Sirois): Scope the extra requirements to the partition. # Right now we just use a global set of extra requirements and these might not be compatible # with all partitions. See: https://github.com/pantsbuild/pants/issues/11556 mypy_extra_requirements_pex_get = Get( Pex, PexRequest( output_filename="mypy_extra_requirements.pex", internal_only=True, requirements=PexRequirements(mypy.extra_requirements), interpreter_constraints=partition.interpreter_constraints, ), ) mypy_pex_get = Get( VenvPex, PexRequest( output_filename="mypy.pex", internal_only=True, main=mypy.main, requirements=PexRequirements((*mypy.all_requirements, *plugin_requirements)), interpreter_constraints=tool_interpreter_constraints, ), ) config_files_get = Get(ConfigFiles, ConfigFilesRequest, mypy.config_request) ( plugin_sources, closure_sources, roots_sources, mypy_pex, requirements_pex, mypy_extra_requirements_pex, config_files, ) = await MultiGet( plugin_sources_get, closure_sources_get, roots_sources_get, mypy_pex_get, requirements_pex_get, mypy_extra_requirements_pex_get, config_files_get, ) python_files = determine_python_files(roots_sources.snapshot.files) file_list_path = "__files.txt" file_list_digest_request = Get( Digest, CreateDigest([FileContent(file_list_path, "\n".join(python_files).encode())]), ) typechecked_venv_pex_request = Get( VenvPex, PexRequest( output_filename="typechecked_venv.pex", internal_only=True, pex_path=[requirements_pex, mypy_extra_requirements_pex], interpreter_constraints=partition.interpreter_constraints, ), ) typechecked_venv_pex, file_list_digest = await MultiGet( typechecked_venv_pex_request, file_list_digest_request ) merged_input_files = await Get( Digest, MergeDigests( [ file_list_digest, plugin_sources.source_files.snapshot.digest, closure_sources.source_files.snapshot.digest, typechecked_venv_pex.digest, config_files.snapshot.digest, ] ), ) all_used_source_roots = sorted( set(itertools.chain(plugin_sources.source_roots, closure_sources.source_roots)) ) env = { "PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots), "MYPYPATH": ":".join(all_used_source_roots), } result = await Get( FallibleProcessResult, VenvPexProcess( mypy_pex, argv=generate_argv( mypy, typechecked_venv_pex, file_list_path=file_list_path, python_version=python_version, ), input_digest=merged_input_files, extra_env=env, description=f"Run MyPy on {pluralize(len(python_files), 'file')}.", level=LogLevel.DEBUG, ), ) return TypecheckResult.from_fallible_process_result( result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)) )
async def create_pex_binary_run_request( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment, python_setup: PythonSetup, ) -> RunRequest: entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in local_dists_get. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest( [field_set.address], output_filename=f"{field_set.address.target_name}.pex", internal_only=True, include_source_files=False, # Note that the file for first-party entry points is not in the PEX itself. In that # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`. main=entry_point.val or field_set.script.value, resolve_and_lockfile=field_set.resolve.resolve_and_lockfile( python_setup), additional_args=( *field_set.generate_additional_args(pex_binary_defaults), # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH # below, it's important for any app that re-executes itself that these environment # variables are not stripped. "--no-strip-pex-env", ), ), ) pex_get = Get(Pex, PexRequest, requirements_pex_request) sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) pex, sources = await MultiGet(pex_get, sources_get) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [field_set.address], internal_only=True, interpreter_constraints=requirements_pex_request. interpreter_constraints, sources=sources, ), ) merged_digest = await Get( Digest, MergeDigests([ pex.digest, local_dists.pex.digest, local_dists.remaining_sources.source_files.snapshot.digest, ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) complete_pex_env = pex_env.in_workspace() args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), "PEX_PATH": in_chroot(local_dists.pex.name), "PEX_EXTRA_SYS_PATH": os.pathsep.join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def mypy_typecheck_partition( partition: MyPyPartition, config_file: MyPyConfigFile, first_party_plugins: MyPyFirstPartyPlugins, mypy: MyPy, python_setup: PythonSetup, ) -> CheckResult: # MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6, # and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with # Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a # loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if # 3.8+ is not installed on the machine. tool_interpreter_constraints = (partition.interpreter_constraints if ( mypy.options.is_default("interpreter_constraints") and partition.interpreter_constraints.requires_python38_or_newer( python_setup.interpreter_universe)) else mypy.interpreter_constraints) closure_sources_get = Get(PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) roots_sources_get = Get( SourceFiles, SourceFilesRequest( tgt.get(PythonSources) for tgt in partition.root_targets)) # See `requirements_venv_pex` for how this will get wrapped in a `VenvPex`. requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in partition.root_targets), hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, ), ) mypy_pex_get = Get( VenvPex, PexRequest( output_filename="mypy.pex", internal_only=True, main=mypy.main, requirements=mypy.pex_requirements( extra_requirements=first_party_plugins.requirement_strings, ), interpreter_constraints=tool_interpreter_constraints, ), ) closure_sources, roots_sources, mypy_pex, requirements_pex = await MultiGet( closure_sources_get, roots_sources_get, mypy_pex_get, requirements_pex_get) python_files = determine_python_files(roots_sources.snapshot.files) file_list_path = "__files.txt" file_list_digest_request = Get( Digest, CreateDigest( [FileContent(file_list_path, "\n".join(python_files).encode())]), ) # This creates a venv with all the 3rd-party requirements used by the code. We tell MyPy to # use this venv by setting `--python-executable`. Note that this Python interpreter is # different than what we run MyPy with. # # We could have directly asked the `PexFromTargetsRequest` to return a `VenvPex`, rather than # `Pex`, but that would mean missing out on sharing a cache with other goals like `test` and # `run`. requirements_venv_pex_request = Get( VenvPex, PexRequest( output_filename="requirements_venv.pex", internal_only=True, pex_path=[requirements_pex], interpreter_constraints=partition.interpreter_constraints, ), ) requirements_venv_pex, file_list_digest = await MultiGet( requirements_venv_pex_request, file_list_digest_request) merged_input_files = await Get( Digest, MergeDigests([ file_list_digest, first_party_plugins.sources_digest, closure_sources.source_files.snapshot.digest, requirements_venv_pex.digest, config_file.digest, ]), ) all_used_source_roots = sorted( set( itertools.chain(first_party_plugins.source_roots, closure_sources.source_roots))) env = { "PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots), "MYPYPATH": ":".join(all_used_source_roots), } result = await Get( FallibleProcessResult, VenvPexProcess( mypy_pex, argv=generate_argv( mypy, venv_python=requirements_venv_pex.python.argv0, file_list_path=file_list_path, python_version=config_file.python_version_to_autoset( partition.interpreter_constraints, python_setup.interpreter_universe), ), input_digest=merged_input_files, extra_env=env, output_directories=(REPORT_DIR, ), description=f"Run MyPy on {pluralize(len(python_files), 'file')}.", level=LogLevel.DEBUG, ), ) report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR)) return CheckResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)), report=report, )
async def pylint_lint_partition(partition: PylintPartition, pylint: Pylint) -> LintResult: requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (field_set.address for field_set in partition.field_sets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition.interpreter_constraints, internal_only=True, direct_deps_only=True, ), ) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in partition.plugin_targets if plugin_tgt.has_field(PythonRequirementsField) ) # Right now any Pylint transitive requirements will shadow corresponding user # requirements, which could lead to problems. pylint_pex_request = Get( Pex, PexRequest( output_filename="pylint.pex", internal_only=True, requirements=PexRequirements([*pylint.all_requirements, *plugin_requirements]), entry_point=pylint.entry_point, interpreter_constraints=partition.interpreter_constraints, # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 additional_args=("--pex-path", requirements_pex_request.input.output_filename), ), ) config_digest_request = Get( Digest, PathGlobs( globs=[pylint.config] if pylint.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--pylint-config`", ), ) prepare_plugin_sources_request = Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(partition.plugin_targets) ) prepare_python_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies) ) field_set_sources_request = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets) ) ( pylint_pex, requirements_pex, config_digest, prepared_plugin_sources, prepared_python_sources, field_set_sources, ) = await MultiGet( pylint_pex_request, requirements_pex_request, config_digest_request, prepare_plugin_sources_request, prepare_python_sources_request, field_set_sources_request, ) prefixed_plugin_sources = ( await Get( Digest, AddPrefix(prepared_plugin_sources.stripped_source_files.snapshot.digest, "__plugins"), ) if pylint.source_plugins else EMPTY_DIGEST ) pythonpath = list(prepared_python_sources.source_roots) if pylint.source_plugins: # NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must # point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) pythonpath.append("__plugins") input_digest = await Get( Digest, MergeDigests( ( pylint_pex.digest, requirements_pex.digest, config_digest, prefixed_plugin_sources, prepared_python_sources.source_files.snapshot.digest, ) ), ) result = await Get( FallibleProcessResult, PexProcess( pylint_pex, argv=generate_args(source_files=field_set_sources, pylint=pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description=f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return LintResult.from_fallible_process_result( result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)) )
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup) requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([request.field_set.address], internal_only=True), ) pytest_pex_get = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), interpreter_constraints=interpreter_constraints, internal_only=True, ), ) extra_output_directory_digest_get = Get( Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)])) prepared_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)) build_package_dependencies_get = Get( BuiltPackageDependencies, BuildPackageDependenciesRequest( request.field_set.runtime_package_dependencies), ) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_get = Get( SourceFiles, SourceFilesRequest([request.field_set.sources])) ( pytest_pex, requirements_pex, prepared_sources, field_set_source_files, built_package_dependencies, extra_output_directory_digest, ) = await MultiGet( pytest_pex_get, requirements_pex_get, prepared_sources_get, field_set_source_files_get, build_package_dependencies_get, extra_output_directory_digest_get, ) pytest_runner_pex_get = Get( VenvPex, PexRequest( output_filename="pytest_runner.pex", interpreter_constraints=interpreter_constraints, main=ConsoleScript("pytest"), internal_only=True, pex_path=[pytest_pex, requirements_pex], ), ) config_files_get = Get( ConfigFiles, ConfigFilesRequest, pytest.config_request(field_set_source_files.snapshot.dirs), ) pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get, config_files_get) input_digest = await Get( Digest, MergeDigests(( coverage_config.digest, prepared_sources.source_files.snapshot.digest, config_files.snapshot.digest, extra_output_directory_digest, *(pkg.digest for pkg in built_package_dependencies), )), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend((f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}")) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: output_files.append(".coverage") cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else ( ".", ) coverage_args = [ "--cov-report=", # Turn off output. f"--cov-config={coverage_config.path}", *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), **test_extra_env.env, } # Cache test runs only if they are successful, or not at all if `--test-force`. cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL process = await Get( Process, VenvPexProcess( pytest_runner_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_directories=(_EXTRA_OUTPUT_DIR, ), output_files=output_files, timeout_seconds=request.field_set.timeout. calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, cache_scope=cache_scope, ), ) return TestSetup(process, results_file_name=results_file_name)