async def package_python_dist( field_set: PythonDistributionFieldSet, python_setup: PythonSetup, ) -> BuiltPackage: transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])) exported_target = ExportedTarget(transitive_targets.roots[0]) interpreter_constraints = PexInterpreterConstraints.create_from_targets( transitive_targets.closure, python_setup ) chroot = await Get( SetupPyChroot, SetupPyChrootRequest(exported_target, py2=interpreter_constraints.includes_python2()), ) # If commands were provided, run setup.py with them; Otherwise just dump chroots. commands = exported_target.target.get(SetupPyCommandsField).value or () if commands: validate_commands(commands) setup_py_result = await Get( RunSetupPyResult, RunSetupPyRequest(exported_target, interpreter_constraints, chroot, commands), ) dist_snapshot = await Get(Snapshot, Digest, setup_py_result.output) return BuiltPackage( setup_py_result.output, tuple(BuiltPackageArtifact(path) for path in dist_snapshot.files), ) else: dirname = f"{chroot.setup_kwargs.name}-{chroot.setup_kwargs.version}" rel_chroot = await Get(Digest, AddPrefix(chroot.digest, dirname)) return BuiltPackage(rel_chroot, (BuiltPackageArtifact(dirname),))
async def infer_python_dependencies_via_imports( request: InferPythonImportDependencies, python_infer_subsystem: PythonInferSubsystem, python_setup: PythonSetup, ) -> InferredDependencies: if not python_infer_subsystem.imports: return InferredDependencies([], sibling_dependencies_inferrable=False) wrapped_tgt = await Get(WrappedTarget, Address, request.sources_field.address) detected_imports = await Get( ParsedPythonImports, ParsePythonImportsRequest( request.sources_field, PexInterpreterConstraints.create_from_targets([wrapped_tgt.target], python_setup), ), ) relevant_imports = (detected_imports.all_imports if python_infer_subsystem.string_imports else detected_imports.explicit_imports) owners_per_import = await MultiGet( Get(PythonModuleOwners, PythonModule(imported_module)) for imported_module in relevant_imports if imported_module not in combined_stdlib) merged_result = sorted( set(itertools.chain.from_iterable(owners_per_import))) return InferredDependencies(merged_result, sibling_dependencies_inferrable=True)
async def mypy_typecheck( request: MyPyRequest, mypy: MyPy, python_setup: PythonSetup ) -> TypecheckResults: if mypy.skip: return TypecheckResults([], typechecker_name="MyPy") # We batch targets by their interpreter constraints to ensure, for example, that all Python 2 # targets run together and all Python 3 targets run together. We can only do this by setting # the `--python-version` option, but we allow the user to set it as a safety valve. We warn if # they've set the option. config_files = await Get(ConfigFiles, ConfigFilesRequest, mypy.config_request) config_content = await Get(DigestContents, Digest, config_files.snapshot.digest) python_version_configured = check_and_warn_if_python_version_configured( config=next(iter(config_content), None), args=mypy.args ) # When determining how to batch by interpreter constraints, we must consider the entire # transitive closure to get the final resulting constraints. # TODO(#10863): Improve the performance of this. transitive_targets_per_field_set = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])) for field_set in request.field_sets ) interpreter_constraints_to_transitive_targets = defaultdict(set) for transitive_targets in transitive_targets_per_field_set: interpreter_constraints = PexInterpreterConstraints.create_from_targets( transitive_targets.closure, python_setup ) or PexInterpreterConstraints(mypy.interpreter_constraints) interpreter_constraints_to_transitive_targets[interpreter_constraints].add( transitive_targets ) partitions = [] for interpreter_constraints, all_transitive_targets in sorted( interpreter_constraints_to_transitive_targets.items() ): combined_roots: OrderedSet[Target] = OrderedSet() combined_closure: OrderedSet[Target] = OrderedSet() for transitive_targets in all_transitive_targets: combined_roots.update(transitive_targets.roots) combined_closure.update(transitive_targets.closure) partitions.append( MyPyPartition( FrozenOrderedSet(combined_roots), FrozenOrderedSet(combined_closure), interpreter_constraints, python_version_already_configured=python_version_configured, ) ) partitioned_results = await MultiGet( Get(TypecheckResult, MyPyPartition, partition) for partition in partitions ) return TypecheckResults(partitioned_results, typechecker_name="MyPy")
async def infer_python_dependencies_via_imports( request: InferPythonImportDependencies, python_infer_subsystem: PythonInferSubsystem, python_setup: PythonSetup, ) -> InferredDependencies: if not python_infer_subsystem.imports: return InferredDependencies([], sibling_dependencies_inferrable=False) wrapped_tgt = await Get(WrappedTarget, Address, request.sources_field.address) explicitly_provided_deps, detected_imports = await MultiGet( Get(ExplicitlyProvidedDependencies, DependenciesRequest(wrapped_tgt.target[Dependencies])), Get( ParsedPythonImports, ParsePythonImportsRequest( request.sources_field, PexInterpreterConstraints.create_from_targets( [wrapped_tgt.target], python_setup), ), ), ) relevant_imports = tuple( imp for imp in (detected_imports.all_imports if python_infer_subsystem. string_imports else detected_imports.explicit_imports) if imp not in combined_stdlib) owners_per_import = await MultiGet( Get(PythonModuleOwners, PythonModule(imported_module)) for imported_module in relevant_imports) merged_result: set[Address] = set() for owners, imp in zip(owners_per_import, relevant_imports): merged_result.update(owners.unambiguous) address = wrapped_tgt.target.address explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference( owners.ambiguous, address, import_reference="module", context=f"The target {address} imports `{imp}`", ) maybe_disambiguated = explicitly_provided_deps.disambiguated_via_ignores( owners.ambiguous) if maybe_disambiguated: merged_result.add(maybe_disambiguated) return InferredDependencies(sorted(merged_result), sibling_dependencies_inferrable=True)
async def pex_from_targets( request: PexFromTargetsRequest, python_setup: PythonSetup, constraints_file: MaybeConstraintsFile, ) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets ) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses) ) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets) ) input_digests.append(prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup ) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or PexInterpreterConstraints( python_setup.interpreter_constraints ) exact_reqs = PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField) ), additional_requirements=request.additional_requirements, ) requirements = exact_reqs repository_pex: Pex | None = None description = request.description if constraints_file.path: constraints_file_contents = await Get(DigestContents, Digest, constraints_file.digest) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=constraints_file.path, ) ) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names url_reqs = set() # E.g., 'foobar@ git+https://github.com/foo/bar.git@branch' name_reqs = set() # E.g., foobar>=1.2.3 name_req_projects = set() for req_str in exact_reqs: req = Requirement.parse(req_str) if req.url: # type: ignore[attr-defined] url_reqs.add(req) else: name_reqs.add(req) name_req_projects.add(canonicalize_project_name(req.project_name)) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } # Constraints files must only contain name reqs, not URL reqs (those are already # constrained by their very nature). See https://github.com/pypa/pip/issues/8210. unconstrained_projects = name_req_projects - constraint_file_projects if unconstrained_projects: constraints_descr = ( f"constraints file {constraints_file.path}" if python_setup.requirement_constraints else f"_python_constraints target {python_setup.requirement_constraints_target}" ) logger.warning( f"The {constraints_descr} does not contain entries for the following " f"requirements: {', '.join(unconstrained_projects)}" ) if python_setup.resolve_all_constraints: if unconstrained_projects: logger.warning( "Ignoring `[python_setup].resolve_all_constraints` option because constraints " "file does not cover all requirements." ) else: # To get a full set of requirements we must add the URL requirements to the # constraints file, since the latter cannot contain URL requirements. # NB: We can only add the URL requirements we know about here, i.e., those that # are transitive deps of the targets in play. There may be others in the repo. # So we may end up creating a few different repository pexes, each with identical # name requirements but different subsets of URL requirements. Fortunately since # all these repository pexes will have identical pinned versions of everything, # this is not a correctness issue, only a performance one. # TODO: Address this as part of providing proper lockfile support. However we # generate lockfiles, they must be able to include URL requirements. all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)} repository_pex = await Get( Pex, PexRequest( description=f"Resolving {python_setup.requirement_constraints}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements(all_constraints), interpreter_constraints=interpreter_constraints, platforms=request.platforms, additional_args=["-vvv"], ), ) elif ( python_setup.resolve_all_constraints and python_setup.resolve_all_constraints_was_set_explicitly() ): raise ValueError( "[python-setup].resolve_all_constraints is enabled, so either " "[python-setup].requirement_constraints or " "[python-setup].requirement_constraints_target must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, main=request.main, sources=merged_input_digest, additional_inputs=request.additional_inputs, repository_pex=repository_pex, additional_args=request.additional_args, description=description, )
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets ) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses) ) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets) ) input_digests.append(prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup ) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or PexInterpreterConstraints( python_setup.interpreter_constraints ) exact_reqs = PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField) ), additional_requirements=request.additional_requirements, ) requirements = exact_reqs description = request.description if python_setup.requirement_constraints: # In requirement strings Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names exact_req_projects = { canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs } constraints_file_contents = await Get( DigestContents, PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin="the option `--python-setup-requirement-constraints`", ), ) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=python_setup.requirement_constraints, ) ) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( f"The constraints file {python_setup.requirement_constraints} does not contain " f"entries for the following requirements: {', '.join(unconstrained_projects)}" ) if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or ( python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES and request.internal_only ): if unconstrained_projects: logger.warning( "Ignoring resolve_all_constraints setting in [python_setup] scope " "because constraints file does not cover all requirements." ) else: requirements = PexRequirements(str(req) for req in constraints_file_reqs) description = description or f"Resolving {python_setup.requirement_constraints}" elif ( python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER and python_setup.resolve_all_constraints_was_set_explicitly() ): raise ValueError( f"[python-setup].resolve_all_constraints is set to " f"{python_setup.resolve_all_constraints.value}, so " f"[python-setup].requirement_constraints must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, entry_point=request.entry_point, sources=merged_input_digest, additional_inputs=request.additional_inputs, additional_args=request.additional_args, description=description, )
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.field_set.address]) ) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup ) requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([request.field_set.address], internal_only=True), ) pytest_pex_request = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), interpreter_constraints=interpreter_constraints, internal_only=True, ), ) prepared_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True) ) # Create any assets that the test depends on through the `runtime_package_dependencies` field. assets: Tuple[BuiltPackage, ...] = () unparsed_runtime_packages = ( request.field_set.runtime_package_dependencies.to_unparsed_address_inputs() ) if unparsed_runtime_packages.values: runtime_package_targets = await Get( Targets, UnparsedAddressInputs, unparsed_runtime_packages ) field_sets_per_target = await Get( FieldSetsPerTarget, FieldSetsPerTargetRequest(PackageFieldSet, runtime_package_targets), ) assets = await MultiGet( Get(BuiltPackage, PackageFieldSet, field_set) for field_set in field_sets_per_target.field_sets ) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_request = Get( SourceFiles, SourceFilesRequest([request.field_set.sources]) ) pytest_pex, requirements_pex, prepared_sources, field_set_source_files = await MultiGet( pytest_pex_request, requirements_pex_request, prepared_sources_request, field_set_source_files_request, ) pytest_runner_pex = await Get( VenvPex, PexRequest( output_filename="pytest_runner.pex", interpreter_constraints=interpreter_constraints, # TODO(John Sirois): Switch to ConsoleScript once Pex supports discovering console # scripts via the PEX_PATH: https://github.com/pantsbuild/pex/issues/1257 main=EntryPoint("pytest"), internal_only=True, pex_path=[pytest_pex, requirements_pex], ), ) input_digest = await Get( Digest, MergeDigests( ( coverage_config.digest, prepared_sources.source_files.snapshot.digest, *(binary.digest for binary in assets), ) ), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend( (f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}") ) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: output_files.append(".coverage") cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else (".",) coverage_args = [ "--cov-report=", # Turn off output. *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), } extra_env.update(test_extra_env.env) # Cache test runs only if they are successful, or not at all if `--test-force`. cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL process = await Get( Process, VenvPexProcess( pytest_runner_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_files=output_files, timeout_seconds=request.field_set.timeout.calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, cache_scope=cache_scope, ), ) return TestSetup(process, results_file_name=results_file_name)
async def pex_from_targets( request: PexFromTargetsRequest, python_setup: PythonSetup, constraints_file: MaybeConstraintsFile, ) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses)) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get(StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets)) input_digests.append( prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or PexInterpreterConstraints( python_setup.interpreter_constraints) exact_reqs = PexRequirements.create_from_requirement_fields( (tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField)), additional_requirements=request.additional_requirements, ) requirements = exact_reqs repository_pex: Pex | None = None description = request.description if constraints_file.path: constraints_file_contents = await Get(DigestContents, Digest, constraints_file.digest) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=constraints_file.path, )) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names exact_req_projects = { canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs } constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: constraints_descr = ( f"constraints file {constraints_file.path}" if python_setup.requirement_constraints else f"_python_constraints target {python_setup.requirement_constraints_target}" ) logger.warning( f"The {constraints_descr} does not contain entries for the following " f"requirements: {', '.join(unconstrained_projects)}") if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or ( python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES and request.internal_only): if unconstrained_projects: logger.warning( "Ignoring `[python_setup].resolve_all_constraints` option because constraints " "file does not cover all requirements.") else: repository_pex = await Get( Pex, PexRequest( description= f"Resolving {python_setup.requirement_constraints}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements( str(req) for req in constraints_file_reqs), interpreter_constraints=interpreter_constraints, platforms=request.platforms, ), ) elif (python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER and python_setup.resolve_all_constraints_was_set_explicitly()): raise ValueError( "[python-setup].resolve_all_constraints is set to " f"{python_setup.resolve_all_constraints.value}, so " "either [python-setup].requirement_constraints or " "[python-setup].requirement_constraints_target must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, main=request.main, sources=merged_input_digest, additional_inputs=request.additional_inputs, repository_pex=repository_pex, additional_args=request.additional_args, description=description, )
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup) requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([request.field_set.address], internal_only=True), ) pytest_pex_get = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), interpreter_constraints=interpreter_constraints, internal_only=True, ), ) extra_output_directory_digest_get = Get( Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)])) prepared_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)) build_package_dependencies_get = Get( BuiltPackageDependencies, BuildPackageDependenciesRequest( request.field_set.runtime_package_dependencies), ) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_get = Get( SourceFiles, SourceFilesRequest([request.field_set.sources])) ( pytest_pex, requirements_pex, prepared_sources, field_set_source_files, built_package_dependencies, extra_output_directory_digest, ) = await MultiGet( pytest_pex_get, requirements_pex_get, prepared_sources_get, field_set_source_files_get, build_package_dependencies_get, extra_output_directory_digest_get, ) pytest_runner_pex_get = Get( VenvPex, PexRequest( output_filename="pytest_runner.pex", interpreter_constraints=interpreter_constraints, main=ConsoleScript("pytest"), internal_only=True, pex_path=[pytest_pex, requirements_pex], ), ) config_files_get = Get( ConfigFiles, ConfigFilesRequest, pytest.config_request(field_set_source_files.snapshot.dirs), ) pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get, config_files_get) input_digest = await Get( Digest, MergeDigests(( coverage_config.digest, prepared_sources.source_files.snapshot.digest, config_files.snapshot.digest, extra_output_directory_digest, *(pkg.digest for pkg in built_package_dependencies), )), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend((f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}")) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: output_files.append(".coverage") cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else ( ".", ) coverage_args = [ "--cov-report=", # Turn off output. f"--cov-config={coverage_config.path}", *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), **test_extra_env.env, } # Cache test runs only if they are successful, or not at all if `--test-force`. cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL process = await Get( Process, VenvPexProcess( pytest_runner_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_directories=(_EXTRA_OUTPUT_DIR, ), output_files=output_files, timeout_seconds=request.field_set.timeout. calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, cache_scope=cache_scope, ), ) return TestSetup(process, results_file_name=results_file_name)
async def py_constraints( addresses: Addresses, console: Console, py_constraints_subsystem: PyConstraintsSubsystem, python_setup: PythonSetup, registered_target_types: RegisteredTargetTypes, union_membership: UnionMembership, ) -> PyConstraintsGoal: if py_constraints_subsystem.summary: if addresses: console.print_stderr( "The `py-constraints --summary` goal does not take file/target arguments. Run " "`help py-constraints` for more details.") return PyConstraintsGoal(exit_code=1) all_expanded_targets, all_explicit_targets = await MultiGet( Get(Targets, AddressSpecs([DescendantAddresses("")])), Get(UnexpandedTargets, AddressSpecs([DescendantAddresses("")])), ) all_python_targets = sorted( { t for t in (*all_expanded_targets, *all_explicit_targets) if t.has_field(InterpreterConstraintsField) }, key=lambda tgt: tgt.address, ) constraints_per_tgt = [ PexInterpreterConstraints.create_from_targets([tgt], python_setup) for tgt in all_python_targets ] transitive_targets_per_tgt = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address])) for tgt in all_python_targets) transitive_constraints_per_tgt = [ PexInterpreterConstraints.create_from_targets( transitive_targets.closure, python_setup) for transitive_targets in transitive_targets_per_tgt ] dependees_per_root = await MultiGet( Get( Dependees, DependeesRequest( [tgt.address], transitive=True, include_roots=False)) for tgt in all_python_targets) data = [{ "Target": tgt.address.spec, "Constraints": str(constraints), "Transitive Constraints": str(transitive_constraints), "# Dependencies": len(transitive_targets.dependencies), "# Dependees": len(dependees), } for tgt, constraints, transitive_constraints, transitive_targets, dependees in zip( all_python_targets, constraints_per_tgt, transitive_constraints_per_tgt, transitive_targets_per_tgt, dependees_per_root, )] with py_constraints_subsystem.output_sink(console) as stdout: writer = csv.DictWriter( stdout, fieldnames=[ "Target", "Constraints", "Transitive Constraints", "# Dependencies", "# Dependees", ], ) writer.writeheader() for entry in data: writer.writerow(entry) return PyConstraintsGoal(exit_code=0) transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(addresses)) final_constraints = PexInterpreterConstraints.create_from_targets( transitive_targets.closure, python_setup) if not final_constraints: target_types_with_constraints = sorted( tgt_type.alias for tgt_type in registered_target_types.types if tgt_type.class_has_field(InterpreterConstraintsField, union_membership)) logger.warning( "No Python files/targets matched for the `py-constraints` goal. All target types with " f"Python interpreter constraints: {', '.join(target_types_with_constraints)}" ) return PyConstraintsGoal(exit_code=0) constraints_to_addresses = defaultdict(set) for tgt in transitive_targets.closure: constraints = PexInterpreterConstraints.create_from_targets( [tgt], python_setup) if not constraints: continue constraints_to_addresses[constraints].add(tgt.address) with py_constraints_subsystem.output(console) as output_stdout: output_stdout(f"Final merged constraints: {final_constraints}\n") if len(addresses) > 1: merged_constraints_warning = ( "(These are the constraints used if you were to depend on all of the input " "files/targets together, even though they may end up never being used together in " "the real world. Consider using a more precise query or running " "`./pants py-constriants --summary`.)\n") output_stdout(indent(fill(merged_constraints_warning, 80), " ")) for constraint, addrs in sorted(constraints_to_addresses.items()): output_stdout(f"\n{constraint}\n") for addr in sorted(addrs): output_stdout(f" {addr}\n") return PyConstraintsGoal(exit_code=0)
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup) # Defaults to zip_safe=False. requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([request.field_set.address], internal_only=True), ) pytest_pex_request = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), interpreter_constraints=interpreter_constraints, entry_point="pytest:main", internal_only=True, additional_args=( # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import # magic directly from zip files. `zipp` has pathologically bad behavior with large # zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. # Long term, it would be better to fix Zipp (whose fix would then need to be used # by importlib_metadata and then by Pytest). See # https://github.com/jaraco/zipp/pull/26. "--not-zip-safe", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 "--pex-path", requirements_pex_request.input.output_filename, ), ), ) prepared_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)) # Create any assets that the test depends on through the `runtime_package_dependencies` field. assets: Tuple[BuiltPackage, ...] = () unparsed_runtime_packages = (request.field_set.runtime_package_dependencies .to_unparsed_address_inputs()) if unparsed_runtime_packages.values: runtime_package_targets = await Get(Targets, UnparsedAddressInputs, unparsed_runtime_packages) field_sets_per_target = await Get( FieldSetsPerTarget, FieldSetsPerTargetRequest(PackageFieldSet, runtime_package_targets), ) assets = await MultiGet( Get(BuiltPackage, PackageFieldSet, field_set) for field_set in field_sets_per_target.field_sets) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_request = Get( SourceFiles, SourceFilesRequest([request.field_set.sources])) pytest_pex, requirements_pex, prepared_sources, field_set_source_files = await MultiGet( pytest_pex_request, requirements_pex_request, prepared_sources_request, field_set_source_files_request, ) input_digest = await Get( Digest, MergeDigests(( coverage_config.digest, prepared_sources.source_files.snapshot.digest, requirements_pex.digest, pytest_pex.digest, *(binary.digest for binary in assets), )), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend((f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}")) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: output_files.append(".coverage") cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else ( ".", ) coverage_args = [ "--cov-report=", # Turn off output. *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), } extra_env.update(test_extra_env.env) # Cache test runs only if they are successful, or not at all if `--test-force`. cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL process = await Get( Process, PexProcess( pytest_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_files=output_files, timeout_seconds=request.field_set.timeout. calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, cache_scope=cache_scope, ), ) return TestSetup(process, results_file_name=results_file_name)