async def package_python_awslambda( field_set: PythonAwsLambdaFieldSet, lambdex: Lambdex ) -> BuiltPackage: output_filename = field_set.output_path.value_or_default( field_set.address, # Lambdas typically use the .zip suffix, so we use that instead of .pex. file_ending="zip", ) # We hardcode the platform value to the appropriate one for each AWS Lambda runtime. # (Running the "hello world" lambda in the example code will report the platform, and can be # used to verify correctness of these platform strings.) py_major, py_minor = field_set.runtime.to_interpreter_version() platform = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}" # set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707 if py_major <= 3 and py_minor < 8: platform += "m" if (py_major, py_minor) == (2, 7): platform += "u" pex_request = TwoStepPexFromTargetsRequest( PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, entry_point=None, output_filename=output_filename, platforms=PexPlatforms([platform]), additional_args=[ # Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels. "--manylinux=manylinux2014", # When we're executing Pex on Linux, allow a local interpreter to be resolved if # available and matching the AMI platform. "--resolve-local-platforms", ], ) ) lambdex_request = PexRequest( output_filename="lambdex.pex", internal_only=True, requirements=PexRequirements(lambdex.all_requirements), interpreter_constraints=PexInterpreterConstraints(lambdex.interpreter_constraints), entry_point=lambdex.entry_point, ) lambdex_pex, pex_result, handler = await MultiGet( Get(Pex, PexRequest, lambdex_request), Get(TwoStepPex, TwoStepPexFromTargetsRequest, pex_request), Get(ResolvedPythonAwsHandler, ResolvePythonAwsHandlerRequest(field_set.handler)), ) input_digest = await Get(Digest, MergeDigests((pex_result.pex.digest, lambdex_pex.digest))) # NB: Lambdex modifies its input pex in-place, so the input file is also the output file. result = await Get( ProcessResult, PexProcess( lambdex_pex, argv=("build", "-e", handler.val, output_filename), input_digest=input_digest, output_files=(output_filename,), description=f"Setting up handler in {output_filename}", ), ) artifact = BuiltPackageArtifact( output_filename, extra_log_lines=( f" Runtime: {field_set.runtime.value}", # The AWS-facing handler function is always lambdex_handler.handler, which is the # wrapper injected by lambdex that manages invocation of the actual handler. " Handler: lambdex_handler.handler", ), ) return BuiltPackage(digest=result.output_digest, artifacts=(artifact,))
async def package_python_awslambda( field_set: PythonAwsLambdaFieldSet, lambdex: Lambdex, union_membership: UnionMembership) -> BuiltPackage: output_filename = field_set.output_path.value_or_default( # Lambdas typically use the .zip suffix, so we use that instead of .pex. file_ending="zip", ) # We hardcode the platform value to the appropriate one for each AWS Lambda runtime. # (Running the "hello world" lambda in the example code will report the platform, and can be # used to verify correctness of these platform strings.) py_major, py_minor = field_set.runtime.to_interpreter_version() platform = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}" # set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707 if py_major <= 3 and py_minor < 8: platform += "m" if (py_major, py_minor) == (2, 7): platform += "u" additional_pex_args = ( # Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels. "--manylinux=manylinux2014", # When we're executing Pex on Linux, allow a local interpreter to be resolved if # available and matching the AMI platform. "--resolve-local-platforms", ) pex_request = PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, output_filename=output_filename, platforms=PexPlatforms([platform]), additional_args=additional_pex_args, additional_lockfile_args=additional_pex_args, ) lambdex_request = PexRequest( output_filename="lambdex.pex", internal_only=True, requirements=lambdex.pex_requirements(), interpreter_constraints=lambdex.interpreter_constraints, main=lambdex.main, ) lambdex_pex, pex_result, handler, transitive_targets = await MultiGet( Get(VenvPex, PexRequest, lambdex_request), Get(Pex, PexFromTargetsRequest, pex_request), Get(ResolvedPythonAwsHandler, ResolvePythonAwsHandlerRequest(field_set.handler)), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Warn if users depend on `files` targets, which won't be included in the PEX and is a common # gotcha. file_tgts = targets_with_sources_types([FileSourceField], transitive_targets.dependencies, union_membership) if file_tgts: files_addresses = sorted(tgt.address.spec for tgt in file_tgts) logger.warning( f"The `python_awslambda` target {field_set.address} transitively depends on the below " "`files` targets, but Pants will not include them in the built Lambda. Filesystem APIs " "like `open()` are not able to load files within the binary itself; instead, they " "read from the current working directory." f"\n\nInstead, use `resources` targets. See {doc_url('resources')}." f"\n\nFiles targets dependencies: {files_addresses}") # NB: Lambdex modifies its input pex in-place, so the input file is also the output file. result = await Get( ProcessResult, VenvPexProcess( lambdex_pex, argv=("build", "-e", handler.val, output_filename), input_digest=pex_result.digest, output_files=(output_filename, ), description=f"Setting up handler in {output_filename}", ), ) artifact = BuiltPackageArtifact( output_filename, extra_log_lines=( f" Runtime: {field_set.runtime.value}", # The AWS-facing handler function is always lambdex_handler.handler, which is the # wrapper injected by lambdex that manages invocation of the actual handler. " Handler: lambdex_handler.handler", ), ) return BuiltPackage(digest=result.output_digest, artifacts=(artifact, ))
async def pex_from_targets( request: PexFromTargetsRequest, python_setup: PythonSetup, constraints_file: MaybeConstraintsFile, ) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses)) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get(StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets)) input_digests.append( prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or PexInterpreterConstraints( python_setup.interpreter_constraints) exact_reqs = PexRequirements.create_from_requirement_fields( (tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField)), additional_requirements=request.additional_requirements, ) requirements = exact_reqs repository_pex: Pex | None = None description = request.description if constraints_file.path: constraints_file_contents = await Get(DigestContents, Digest, constraints_file.digest) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=constraints_file.path, )) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names exact_req_projects = { canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs } constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: constraints_descr = ( f"constraints file {constraints_file.path}" if python_setup.requirement_constraints else f"_python_constraints target {python_setup.requirement_constraints_target}" ) logger.warning( f"The {constraints_descr} does not contain entries for the following " f"requirements: {', '.join(unconstrained_projects)}") if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or ( python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES and request.internal_only): if unconstrained_projects: logger.warning( "Ignoring `[python_setup].resolve_all_constraints` option because constraints " "file does not cover all requirements.") else: repository_pex = await Get( Pex, PexRequest( description= f"Resolving {python_setup.requirement_constraints}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements( str(req) for req in constraints_file_reqs), interpreter_constraints=interpreter_constraints, platforms=request.platforms, ), ) elif (python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER and python_setup.resolve_all_constraints_was_set_explicitly()): raise ValueError( "[python-setup].resolve_all_constraints is set to " f"{python_setup.resolve_all_constraints.value}, so " "either [python-setup].requirement_constraints or " "[python-setup].requirement_constraints_target must also be provided." ) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, main=request.main, sources=merged_input_digest, additional_inputs=request.additional_inputs, repository_pex=repository_pex, additional_args=request.additional_args, description=description, )
async def generate_lockfile( req: PythonLockfileRequest, poetry_subsystem: PoetrySubsystem, generate_lockfiles_subsystem: GenerateLockfilesSubsystem, ) -> PythonLockfile: pyproject_toml = create_pyproject_toml(req.requirements, req.interpreter_constraints).encode() pyproject_toml_digest, launcher_digest = await MultiGet( Get(Digest, CreateDigest([FileContent("pyproject.toml", pyproject_toml)])), Get(Digest, CreateDigest([POETRY_LAUNCHER])), ) poetry_pex = await Get( VenvPex, PexRequest( output_filename="poetry.pex", internal_only=True, requirements=poetry_subsystem.pex_requirements(), interpreter_constraints=poetry_subsystem.interpreter_constraints, main=EntryPoint(PurePath(POETRY_LAUNCHER.path).stem), sources=launcher_digest, ), ) # WONTFIX(#12314): Wire up Poetry to named_caches. # WONTFIX(#12314): Wire up all the pip options like indexes. poetry_lock_result = await Get( ProcessResult, VenvPexProcess( poetry_pex, argv=("lock",), input_digest=pyproject_toml_digest, output_files=("poetry.lock", "pyproject.toml"), description=req._description or f"Generate lockfile for {req.resolve_name}", # Instead of caching lockfile generation with LMDB, we instead use the invalidation # scheme from `lockfile_metadata.py` to check for stale/invalid lockfiles. This is # necessary so that our invalidation is resilient to deleting LMDB or running on a # new machine. # # We disable caching with LMDB so that when you generate a lockfile, you always get # the most up-to-date snapshot of the world. This is generally desirable and also # necessary to avoid an awkward edge case where different developers generate different # lockfiles even when generating at the same time. See # https://github.com/pantsbuild/pants/issues/12591. cache_scope=ProcessCacheScope.PER_SESSION, ), ) poetry_export_result = await Get( ProcessResult, VenvPexProcess( poetry_pex, argv=("export", "-o", req.lockfile_dest), input_digest=poetry_lock_result.output_digest, output_files=(req.lockfile_dest,), description=( f"Exporting Poetry lockfile to requirements.txt format for {req.resolve_name}" ), level=LogLevel.DEBUG, ), ) initial_lockfile_digest_contents = await Get( DigestContents, Digest, poetry_export_result.output_digest ) # TODO(#12314) Improve error message on `Requirement.parse` metadata = LockfileMetadata.new( req.interpreter_constraints, {Requirement.parse(i) for i in req.requirements}, ) lockfile_with_header = metadata.add_header_to_lockfile( initial_lockfile_digest_contents[0].content, regenerate_command=( generate_lockfiles_subsystem.custom_command or req._regenerate_command or f"./pants generate-lockfiles --resolve={req.resolve_name}" ), ) final_lockfile_digest = await Get( Digest, CreateDigest([FileContent(req.lockfile_dest, lockfile_with_header)]) ) return PythonLockfile(final_lockfile_digest, req.resolve_name, req.lockfile_dest)
async def mypy_typecheck_partition( partition: MyPyPartition, config_file: MyPyConfigFile, first_party_plugins: MyPyFirstPartyPlugins, mypy: MyPy, python_setup: PythonSetup, ) -> CheckResult: # MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6, # and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with # Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a # loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if # 3.8+ is not installed on the machine. tool_interpreter_constraints = (partition.interpreter_constraints if ( mypy.options.is_default("interpreter_constraints") and partition.interpreter_constraints.requires_python38_or_newer( python_setup.interpreter_universe)) else mypy.interpreter_constraints) closure_sources_get = Get(PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) roots_sources_get = Get( SourceFiles, SourceFilesRequest( tgt.get(PythonSources) for tgt in partition.root_targets)) # See `requirements_venv_pex` for how this will get wrapped in a `VenvPex`. requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in partition.root_targets), hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, ), ) mypy_pex_get = Get( VenvPex, PexRequest( output_filename="mypy.pex", internal_only=True, main=mypy.main, requirements=mypy.pex_requirements( extra_requirements=first_party_plugins.requirement_strings, ), interpreter_constraints=tool_interpreter_constraints, ), ) closure_sources, roots_sources, mypy_pex, requirements_pex = await MultiGet( closure_sources_get, roots_sources_get, mypy_pex_get, requirements_pex_get) python_files = determine_python_files(roots_sources.snapshot.files) file_list_path = "__files.txt" file_list_digest_request = Get( Digest, CreateDigest( [FileContent(file_list_path, "\n".join(python_files).encode())]), ) # This creates a venv with all the 3rd-party requirements used by the code. We tell MyPy to # use this venv by setting `--python-executable`. Note that this Python interpreter is # different than what we run MyPy with. # # We could have directly asked the `PexFromTargetsRequest` to return a `VenvPex`, rather than # `Pex`, but that would mean missing out on sharing a cache with other goals like `test` and # `run`. requirements_venv_pex_request = Get( VenvPex, PexRequest( output_filename="requirements_venv.pex", internal_only=True, pex_path=[requirements_pex], interpreter_constraints=partition.interpreter_constraints, ), ) requirements_venv_pex, file_list_digest = await MultiGet( requirements_venv_pex_request, file_list_digest_request) merged_input_files = await Get( Digest, MergeDigests([ file_list_digest, first_party_plugins.sources_digest, closure_sources.source_files.snapshot.digest, requirements_venv_pex.digest, config_file.digest, ]), ) all_used_source_roots = sorted( set( itertools.chain(first_party_plugins.source_roots, closure_sources.source_roots))) env = { "PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots), "MYPYPATH": ":".join(all_used_source_roots), } result = await Get( FallibleProcessResult, VenvPexProcess( mypy_pex, argv=generate_argv( mypy, venv_python=requirements_venv_pex.python.argv0, file_list_path=file_list_path, python_version=config_file.python_version_to_autoset( partition.interpreter_constraints, python_setup.interpreter_universe), ), input_digest=merged_input_files, extra_env=env, output_directories=(REPORT_DIR, ), description=f"Run MyPy on {pluralize(len(python_files), 'file')}.", level=LogLevel.DEBUG, ), ) report = await Get(Digest, RemovePrefix(result.output_digest, REPORT_DIR)) return CheckResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)), report=report, )
async def setup_black( setup_request: SetupRequest, black: Black, python_setup: PythonSetup ) -> Setup: # Black requires 3.6+ but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6, and 3.7. # However, typed-ast does not understand 3.8+, so instead we must run Black with Python 3.8+ # when relevant. We only do this if if <3.8 can't be used, as we don't want a loose requirement # like `>=3.6` to result in requiring Python 3.8, which would error if 3.8 is not installed on # the machine. all_interpreter_constraints = InterpreterConstraints.create_from_compatibility_fields( (field_set.interpreter_constraints for field_set in setup_request.request.field_sets), python_setup, ) tool_interpreter_constraints = ( all_interpreter_constraints if ( black.options.is_default("interpreter_constraints") and all_interpreter_constraints.requires_python38_or_newer( python_setup.interpreter_universe ) ) else black.interpreter_constraints ) black_pex_get = Get( VenvPex, PexRequest( output_filename="black.pex", internal_only=True, requirements=black.pex_requirements(), interpreter_constraints=tool_interpreter_constraints, main=black.main, ), ) source_files_get = Get( SourceFiles, SourceFilesRequest(field_set.source for field_set in setup_request.request.field_sets), ) source_files, black_pex = await MultiGet(source_files_get, black_pex_get) source_files_snapshot = ( source_files.snapshot if setup_request.request.prior_formatter_result is None else setup_request.request.prior_formatter_result ) config_files = await Get( ConfigFiles, ConfigFilesRequest, black.config_request(source_files_snapshot.dirs) ) input_digest = await Get( Digest, MergeDigests((source_files_snapshot.digest, config_files.snapshot.digest)) ) process = await Get( Process, VenvPexProcess( black_pex, argv=generate_argv(source_files, black, check_only=setup_request.check_only), input_digest=input_digest, output_files=source_files_snapshot.files, description=f"Run Black on {pluralize(len(setup_request.request.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return Setup(process, original_digest=source_files_snapshot.digest)
async def mypy_typecheck_partition(partition: MyPyPartition, mypy: MyPy) -> TypecheckResult: plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, mypy.source_plugins) plugin_transitive_targets_request = Get( TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses)) plugin_transitive_targets, launcher_script = await MultiGet( plugin_transitive_targets_request, Get(Digest, CreateDigest([LAUNCHER_FILE]))) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in plugin_transitive_targets.closure if plugin_tgt.has_field(PythonRequirementsField)) # If the user did not set `--python-version` already, we set it ourselves based on their code's # interpreter constraints. This determines what AST is used by MyPy. python_version = ( None if partition.python_version_already_configured else partition.interpreter_constraints.minimum_python_version()) # MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6, # and 3.7. However, typed-ast does not understand 3.8, so instead we must run MyPy with # Python 3.8 when relevant. We only do this if <3.8 can't be used, as we don't want a # loose requirement like `>=3.6` to result in requiring Python 3.8, which would error if # 3.8 is not installed on the machine. tool_interpreter_constraints = PexInterpreterConstraints(( "CPython>=3.8", ) if ( mypy.options.is_default("interpreter_constraints") and partition.interpreter_constraints.requires_python38_or_newer() ) else mypy.interpreter_constraints) plugin_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(plugin_transitive_targets.closure)) typechecked_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.closure)) # Normally, this `requirements.pex` would be merged with mypy.pex via `--pex-path`. However, # this will cause a runtime error if the interpreter constraints are different between the # PEXes and they have incompatible wheels. # # Instead, we teach MyPy about the requirements by extracting the distributions from # requirements.pex and setting EXTRACTED_WHEELS, which our custom launcher script then # looks for. # # Conventionally, MyPy users might instead set `MYPYPATH` for this. However, doing this # results in type checking the requirements themselves. requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (addr for addr in partition.field_set_addresses), hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, ), ) mypy_pex_request = Get( Pex, PexRequest( output_filename="mypy.pex", internal_only=True, sources=launcher_script, requirements=PexRequirements( itertools.chain(mypy.all_requirements, plugin_requirements)), interpreter_constraints=tool_interpreter_constraints, entry_point=PurePath(LAUNCHER_FILE.path).stem, ), ) config_digest_request = Get(Digest, PathGlobs, config_path_globs(mypy)) ( plugin_sources, typechecked_sources, mypy_pex, requirements_pex, config_digest, ) = await MultiGet( plugin_sources_request, typechecked_sources_request, mypy_pex_request, requirements_pex_request, config_digest_request, ) typechecked_srcs_snapshot = typechecked_sources.source_files.snapshot file_list_path = "__files.txt" python_files = "\n".join( determine_python_files( typechecked_sources.source_files.snapshot.files)) create_file_list_request = Get( Digest, CreateDigest([FileContent(file_list_path, python_files.encode())]), ) file_list_digest, extracted_pex_distributions = await MultiGet( create_file_list_request, Get(ExtractedPexDistributions, Pex, requirements_pex)) merged_input_files = await Get( Digest, MergeDigests([ file_list_digest, plugin_sources.source_files.snapshot.digest, typechecked_srcs_snapshot.digest, mypy_pex.digest, extracted_pex_distributions.digest, config_digest, ]), ) all_used_source_roots = sorted( set( itertools.chain(plugin_sources.source_roots, typechecked_sources.source_roots))) env = { "PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots), "EXTRACTED_WHEELS": ":".join(extracted_pex_distributions.wheel_directory_paths), } result = await Get( FallibleProcessResult, PexProcess( mypy_pex, argv=generate_argv(mypy, file_list_path=file_list_path, python_version=python_version), input_digest=merged_input_files, extra_env=env, description= f"Run MyPy on {pluralize(len(typechecked_srcs_snapshot.files), 'file')}.", level=LogLevel.DEBUG, ), ) return TypecheckResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)))
async def build_local_dists( request: LocalDistsPexRequest, ) -> LocalDistsPex: transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)) applicable_targets = [ tgt for tgt in transitive_targets.closure if PythonDistributionFieldSet.is_applicable(tgt) ] python_dist_field_sets = [ PythonDistributionFieldSet.create(target) for target in applicable_targets ] dists = await MultiGet( [Get(BuiltPackage, PackageFieldSet, field_set) for field_set in python_dist_field_sets] ) # The primary use-case of the "local dists" feature is to support consuming native extensions # as wheels without having to publish them first. # It doesn't seem very useful to consume locally-built sdists, and it makes it hard to # reason about possible sys.path collisions between the in-repo sources and whatever the # sdist will place on the sys.path when it's installed. # So for now we simply ignore sdists, with a warning if necessary. provided_files = set() wheels = [] all_contents = await MultiGet(Get(DigestContents, Digest, dist.digest) for dist in dists) for dist, contents, tgt in zip(dists, all_contents, applicable_targets): artifacts = {(a.relpath or "") for a in dist.artifacts} # A given local dist might build a wheel and an sdist (and maybe other artifacts - # we don't know what setup command was run...) # As long as there is a wheel, we can ignore the other artifacts. wheel = next((art for art in artifacts if art.endswith(".whl")), None) if wheel: wheel_content = next(content for content in contents if content.path == wheel) wheels.append(wheel) buf = BytesIO() buf.write(wheel_content.content) buf.seek(0) with zipfile.ZipFile(buf) as zf: provided_files.update(zf.namelist()) else: logger.warning( f"Encountered a dependency on the {tgt.alias} target at {tgt.address.spec}, but " "this target does not produce a Python wheel artifact. Therefore this target's " "code will be used directly from sources, without a distribution being built, " "and therefore any native extensions in it will not be built.\n\n" f"See {doc_url('python-distributions')} for details on how to set up a {tgt.alias} " "target to produce a wheel." ) dists_digest = await Get(Digest, MergeDigests([dist.digest for dist in dists])) wheels_digest = await Get(Digest, DigestSubset(dists_digest, PathGlobs(["**/*.whl"]))) dists_pex = await Get( Pex, PexRequest( output_filename="local_dists.pex", requirements=PexRequirements(wheels), interpreter_constraints=request.interpreter_constraints, additional_inputs=wheels_digest, internal_only=request.internal_only, ), ) # We check source roots in reverse lexicographic order, # so we'll find the innermost root that matches. source_roots = list(reversed(sorted(request.sources.source_roots))) remaining_sources = set(request.sources.source_files.files) unrooted_files_set = set(request.sources.source_files.unrooted_files) for source in request.sources.source_files.files: if source not in unrooted_files_set: for source_root in source_roots: source_relpath = fast_relpath_optional(source, source_root) if source_relpath is not None and source_relpath in provided_files: remaining_sources.remove(source) remaining_sources_snapshot = await Get( Snapshot, DigestSubset( request.sources.source_files.snapshot.digest, PathGlobs(sorted(remaining_sources)) ), ) subtracted_sources = PythonSourceFiles( SourceFiles(remaining_sources_snapshot, request.sources.source_files.unrooted_files), request.sources.source_roots, ) return LocalDistsPex(dists_pex, subtracted_sources)
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets ) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses) ) all_targets = transitive_targets.closure if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = InterpreterConstraints.create_from_targets( all_targets, python_setup ) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or InterpreterConstraints( python_setup.interpreter_constraints ) sources_digests = [] if request.additional_sources: sources_digests.append(request.additional_sources) if request.include_source_files: sources = await Get(PythonSourceFiles, PythonSourceFilesRequest(all_targets)) else: sources = PythonSourceFiles.empty() additional_inputs_digests = [] if request.additional_inputs: additional_inputs_digests.append(request.additional_inputs) additional_args = request.additional_args if request.include_local_dists: # Note that LocalDistsPexRequest has no `direct_deps_only` mode, so we will build all # local dists in the transitive closure even if the request was for direct_deps_only. # Since we currently use `direct_deps_only` in one case (building a requirements pex # when running pylint) and in that case include_local_dists=False, this seems harmless. local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( request.addresses, internal_only=request.internal_only, interpreter_constraints=interpreter_constraints, sources=sources, ), ) remaining_sources = local_dists.remaining_sources additional_inputs_digests.append(local_dists.pex.digest) additional_args += ("--requirements-pex", local_dists.pex.name) else: remaining_sources = sources remaining_sources_stripped = await Get( StrippedPythonSourceFiles, PythonSourceFiles, remaining_sources ) sources_digests.append(remaining_sources_stripped.stripped_source_files.snapshot.digest) merged_sources_digest, additional_inputs = await MultiGet( Get(Digest, MergeDigests(sources_digests)), Get(Digest, MergeDigests(additional_inputs_digests)), ) requirements = PexRequirements.create_from_requirement_fields( ( tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField) ), additional_requirements=request.additional_requirements, apply_constraints=True, ) description = request.description if requirements: repository_pex: Pex | None = None if python_setup.requirement_constraints: maybe_constraints_repository_pex = await Get( _ConstraintsRepositoryPex, _ConstraintsRepositoryPexRequest( requirements, request.platforms, interpreter_constraints, request.internal_only, request.additional_lockfile_args, ), ) if maybe_constraints_repository_pex.maybe_pex: repository_pex = maybe_constraints_repository_pex.maybe_pex elif ( python_setup.resolve_all_constraints and python_setup.resolve_all_constraints_was_set_explicitly() ): raise ValueError( "`[python].resolve_all_constraints` is enabled, so " "`[python].requirement_constraints` must also be set." ) elif request.resolve_and_lockfile: resolve, lockfile = request.resolve_and_lockfile repository_pex = await Get( Pex, PexRequest( description=f"Installing {lockfile} for the resolve `{resolve}`", output_filename=f"{path_safe(resolve)}_lockfile.pex", internal_only=request.internal_only, requirements=Lockfile( file_path=lockfile, file_path_description_of_origin=( f"the resolve `{resolve}` (from " "`[python].experimental_resolves_to_lockfiles`)" ), # TODO(#12314): Hook up lockfile staleness check. lockfile_hex_digest=None, req_strings=None, ), interpreter_constraints=interpreter_constraints, platforms=request.platforms, additional_args=request.additional_lockfile_args, ), ) elif python_setup.lockfile: repository_pex = await Get( Pex, PexRequest( description=f"Installing {python_setup.lockfile}", output_filename="lockfile.pex", internal_only=request.internal_only, requirements=Lockfile( file_path=python_setup.lockfile, file_path_description_of_origin=( "the option `[python].experimental_lockfile`" ), # TODO(#12314): Hook up lockfile staleness check once multiple lockfiles # are supported. lockfile_hex_digest=None, req_strings=None, ), interpreter_constraints=interpreter_constraints, platforms=request.platforms, additional_args=request.additional_lockfile_args, ), ) requirements = dataclasses.replace(requirements, repository_pex=repository_pex) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, main=request.main, sources=merged_sources_digest, additional_inputs=additional_inputs, additional_args=additional_args, description=description, )
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup) # Defaults to zip_safe=False. requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([request.field_set.address], internal_only=True), ) pytest_pex_request = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), interpreter_constraints=interpreter_constraints, entry_point="pytest:main", internal_only=True, additional_args=( # NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses # `importlib_metadata` and thus `zipp`, does not play nicely when doing import # magic directly from zip files. `zipp` has pathologically bad behavior with large # zipfiles. # TODO: this does have a performance cost as the pex must now be expanded to disk. # Long term, it would be better to fix Zipp (whose fix would then need to be used # by importlib_metadata and then by Pytest). See # https://github.com/jaraco/zipp/pull/26. "--not-zip-safe", # TODO(John Sirois): Support shading python binaries: # https://github.com/pantsbuild/pants/issues/9206 "--pex-path", requirements_pex_request.input.output_filename, ), ), ) prepared_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)) # Create any assets that the test depends on through the `runtime_package_dependencies` field. assets: Tuple[BuiltPackage, ...] = () unparsed_runtime_packages = (request.field_set.runtime_package_dependencies .to_unparsed_address_inputs()) if unparsed_runtime_packages.values: runtime_package_targets = await Get(Targets, UnparsedAddressInputs, unparsed_runtime_packages) field_sets_per_target = await Get( FieldSetsPerTarget, FieldSetsPerTargetRequest(PackageFieldSet, runtime_package_targets), ) assets = await MultiGet( Get(BuiltPackage, PackageFieldSet, field_set) for field_set in field_sets_per_target.field_sets) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_request = Get( SourceFiles, SourceFilesRequest([request.field_set.sources])) pytest_pex, requirements_pex, prepared_sources, field_set_source_files = await MultiGet( pytest_pex_request, requirements_pex_request, prepared_sources_request, field_set_source_files_request, ) input_digest = await Get( Digest, MergeDigests(( coverage_config.digest, prepared_sources.source_files.snapshot.digest, requirements_pex.digest, pytest_pex.digest, *(binary.digest for binary in assets), )), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend((f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}")) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: output_files.append(".coverage") cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else ( ".", ) coverage_args = [ "--cov-report=", # Turn off output. *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), } extra_env.update(test_extra_env.env) # Cache test runs only if they are successful, or not at all if `--test-force`. cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL process = await Get( Process, PexProcess( pytest_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_files=output_files, timeout_seconds=request.field_set.timeout. calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, cache_scope=cache_scope, ), ) return TestSetup(process, results_file_name=results_file_name)
async def generate_python_from_protobuf( request: GeneratePythonFromProtobufRequest, protoc: Protoc, grpc_python_plugin: GrpcPythonPlugin, python_protobuf_subsystem: PythonProtobufSubsystem, ) -> GeneratedSources: download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)) output_dir = "_generated_files" create_output_dir_request = Get(Digest, CreateDigest([Directory(output_dir)])) # Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't # actually generate those dependencies; it only needs to look at their .proto files to work # with imports. transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.protocol_target.address])) # NB: By stripping the source roots, we avoid having to set the value `--proto_path` # for Protobuf imports to be discoverable. all_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest( (tgt.get(Sources) for tgt in transitive_targets.closure), for_sources_types=(ProtobufSources, ), ), ) target_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest([request.protocol_target[ProtobufSources]])) ( downloaded_protoc_binary, empty_output_dir, all_sources_stripped, target_sources_stripped, ) = await MultiGet( download_protoc_request, create_output_dir_request, all_stripped_sources_request, target_stripped_sources_request, ) protoc_gen_mypy_script = "protoc-gen-mypy" mypy_pex = None if python_protobuf_subsystem.mypy_plugin: mypy_pex = await Get( VenvPex, VenvPexRequest( bin_names=[protoc_gen_mypy_script], pex_request=PexRequest( output_filename="mypy_protobuf.pex", internal_only=True, requirements=PexRequirements( [python_protobuf_subsystem.mypy_plugin_version]), # TODO(John Sirois): Fix these interpreter constraints to track the actual # python requirement of the mypy_plugin_version or else plumb an option for # manually setting the constraint to track what mypy_plugin_version needs: # https://github.com/pantsbuild/pants/issues/11565 # Here we guess a constraint that will likely work with any mypy_plugin_version # selected. interpreter_constraints=PexInterpreterConstraints( ["CPython>=3.5"]), ), ), ) downloaded_grpc_plugin = (await Get( DownloadedExternalTool, ExternalToolRequest, grpc_python_plugin.get_request(Platform.current), ) if request.protocol_target.get(ProtobufGrpcToggle).value else None) unmerged_digests = [ all_sources_stripped.snapshot.digest, downloaded_protoc_binary.digest, empty_output_dir, ] if mypy_pex: unmerged_digests.append(mypy_pex.digest) if downloaded_grpc_plugin: unmerged_digests.append(downloaded_grpc_plugin.digest) input_digest = await Get(Digest, MergeDigests(unmerged_digests)) argv = [downloaded_protoc_binary.exe, "--python_out", output_dir] if mypy_pex: argv.extend([ f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}", "--mypy_out", output_dir, ]) if downloaded_grpc_plugin: argv.extend([ f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}", "--grpc_out", output_dir ]) argv.extend(target_sources_stripped.snapshot.files) result = await Get( ProcessResult, Process( argv, input_digest=input_digest, description= f"Generating Python sources from {request.protocol_target.address}.", level=LogLevel.DEBUG, output_directories=(output_dir, ), ), ) # We must do some path manipulation on the output digest for it to look like normal sources, # including adding back a source root. py_source_root = request.protocol_target.get(PythonSourceRootField).value if py_source_root: # Verify that the python source root specified by the target is in fact a source root. source_root_request = SourceRootRequest(PurePath(py_source_root)) else: # The target didn't specify a python source root, so use the protobuf_library's source root. source_root_request = SourceRootRequest.for_target( request.protocol_target) normalized_digest, source_root = await MultiGet( Get(Digest, RemovePrefix(result.output_digest, output_dir)), Get(SourceRoot, SourceRootRequest, source_root_request), ) source_root_restored = (await Get( Snapshot, AddPrefix(normalized_digest, source_root.path)) if source_root.path != "." else await Get( Snapshot, Digest, normalized_digest)) return GeneratedSources(source_root_restored)
async def create_pex_from_targets(request: PexFromTargetsRequest) -> PexRequest: interpreter_constraints = await Get( InterpreterConstraints, InterpreterConstraintsRequest, request.to_interpreter_constraints_request(), ) transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(request.addresses)) sources_digests = [] if request.additional_sources: sources_digests.append(request.additional_sources) if request.include_source_files: sources = await Get(PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure)) else: sources = PythonSourceFiles.empty() additional_inputs_digests = [] if request.additional_inputs: additional_inputs_digests.append(request.additional_inputs) additional_args = request.additional_args if request.include_local_dists: local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( request.addresses, internal_only=request.internal_only, interpreter_constraints=interpreter_constraints, sources=sources, ), ) remaining_sources = local_dists.remaining_sources additional_inputs_digests.append(local_dists.pex.digest) additional_args += ("--requirements-pex", local_dists.pex.name) else: remaining_sources = sources remaining_sources_stripped = await Get( StrippedPythonSourceFiles, PythonSourceFiles, remaining_sources ) sources_digests.append(remaining_sources_stripped.stripped_source_files.snapshot.digest) merged_sources_digest, additional_inputs = await MultiGet( Get(Digest, MergeDigests(sources_digests)), Get(Digest, MergeDigests(additional_inputs_digests)), ) description = request.description if request.include_requirements: requirements = await Get(PexRequirements, _PexRequirementsRequest(request.addresses)) else: requirements = PexRequirements() if requirements: repository_pex = await Get( OptionalPex, _RepositoryPexRequest( request.addresses, requirements=requirements, hardcoded_interpreter_constraints=request.hardcoded_interpreter_constraints, platforms=request.platforms, complete_platforms=request.complete_platforms, internal_only=request.internal_only, additional_lockfile_args=request.additional_lockfile_args, ), ) requirements = dataclasses.replace(requirements, repository_pex=repository_pex.maybe_pex) return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, layout=request.layout, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, complete_platforms=request.complete_platforms, main=request.main, sources=merged_sources_digest, additional_inputs=additional_inputs, additional_args=additional_args, description=description, )
async def pylint_lint_partition(partition: PylintPartition, pylint: Pylint) -> LintResult: requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (field_set.address for field_set in partition.field_sets), # NB: These constraints must be identical to the other PEXes. Otherwise, we risk using # a different version for the requirements than the other two PEXes, which can result # in a PEX runtime error about missing dependencies. hardcoded_interpreter_constraints=partition. interpreter_constraints, internal_only=True, direct_deps_only=True, ), ) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in partition.plugin_targets if plugin_tgt.has_field(PythonRequirementsField)) pylint_pex_get = Get( Pex, PexRequest( output_filename="pylint.pex", internal_only=True, requirements=PexRequirements( [*pylint.all_requirements, *plugin_requirements]), interpreter_constraints=partition.interpreter_constraints, ), ) prepare_plugin_sources_get = Get( StrippedPythonSourceFiles, PythonSourceFilesRequest(partition.plugin_targets)) prepare_python_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.targets_with_dependencies)) field_set_sources_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)) ( pylint_pex, requirements_pex, prepared_plugin_sources, prepared_python_sources, field_set_sources, ) = await MultiGet( pylint_pex_get, requirements_pex_get, prepare_plugin_sources_get, prepare_python_sources_get, field_set_sources_get, ) pylint_runner_pex, config_files = await MultiGet( Get( VenvPex, PexRequest( output_filename="pylint_runner.pex", interpreter_constraints=partition.interpreter_constraints, main=pylint.main, internal_only=True, pex_path=[pylint_pex, requirements_pex], ), ), Get(ConfigFiles, ConfigFilesRequest, pylint.config_request(field_set_sources.snapshot.dirs)), ) prefixed_plugin_sources = (await Get( Digest, AddPrefix( prepared_plugin_sources.stripped_source_files.snapshot.digest, "__plugins"), ) if pylint.source_plugins else EMPTY_DIGEST) pythonpath = list(prepared_python_sources.source_roots) if pylint.source_plugins: # NB: Pylint source plugins must be explicitly loaded via PEX_EXTRA_SYS_PATH. The value must # point to the plugin's directory, rather than to a parent's directory, because # `load-plugins` takes a module name rather than a path to the module; i.e. `plugin`, but # not `path.to.plugin`. (This means users must have specified the parent directory as a # source root.) pythonpath.append("__plugins") input_digest = await Get( Digest, MergeDigests(( config_files.snapshot.digest, prefixed_plugin_sources, prepared_python_sources.source_files.snapshot.digest, )), ) result = await Get( FallibleProcessResult, VenvPexProcess( pylint_runner_pex, argv=generate_argv(field_set_sources, pylint), input_digest=input_digest, extra_env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, description= f"Run Pylint on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)))
async def mypy_typecheck_partition(partition: MyPyPartition, mypy: MyPy) -> TypecheckResult: plugin_target_addresses = await Get(Addresses, UnparsedAddressInputs, mypy.source_plugins) plugin_transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(plugin_target_addresses) ) plugin_requirements = PexRequirements.create_from_requirement_fields( plugin_tgt[PythonRequirementsField] for plugin_tgt in plugin_transitive_targets.closure if plugin_tgt.has_field(PythonRequirementsField) ) # If the user did not set `--python-version` already, we set it ourselves based on their code's # interpreter constraints. This determines what AST is used by MyPy. python_version = ( None if partition.python_version_already_configured else partition.interpreter_constraints.minimum_python_version() ) # MyPy requires 3.5+ to run, but uses the typed-ast library to work with 2.7, 3.4, 3.5, 3.6, # and 3.7. However, typed-ast does not understand 3.8+, so instead we must run MyPy with # Python 3.8+ when relevant. We only do this if <3.8 can't be used, as we don't want a # loose requirement like `>=3.6` to result in requiring Python 3.8+, which would error if # 3.8+ is not installed on the machine. tool_interpreter_constraints = ( partition.interpreter_constraints if ( mypy.options.is_default("interpreter_constraints") and partition.interpreter_constraints.requires_python38_or_newer() ) else PexInterpreterConstraints(mypy.interpreter_constraints) ) plugin_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(plugin_transitive_targets.closure) ) typechecked_sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(partition.closure) ) requirements_pex_request = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (addr for addr in partition.field_set_addresses), hardcoded_interpreter_constraints=partition.interpreter_constraints, internal_only=True, ), ) # TODO(John Sirois): Scope the extra requirements to the partition. # Right now we just use a global set of extra requirements and these might not be compatible # with all partitions. See: https://github.com/pantsbuild/pants/issues/11556 mypy_extra_requirements_pex_request = Get( Pex, PexRequest( output_filename="mypy_extra_requirements.pex", internal_only=True, requirements=PexRequirements(mypy.extra_requirements), interpreter_constraints=partition.interpreter_constraints, ), ) mypy_pex_request = Get( VenvPex, PexRequest( output_filename="mypy.pex", internal_only=True, main=mypy.main, requirements=PexRequirements((*mypy.all_requirements, *plugin_requirements)), interpreter_constraints=tool_interpreter_constraints, ), ) config_digest_request = Get(Digest, PathGlobs, config_path_globs(mypy)) ( plugin_sources, typechecked_sources, mypy_pex, requirements_pex, mypy_extra_requirements_pex, config_digest, ) = await MultiGet( plugin_sources_request, typechecked_sources_request, mypy_pex_request, requirements_pex_request, mypy_extra_requirements_pex_request, config_digest_request, ) typechecked_srcs_snapshot = typechecked_sources.source_files.snapshot file_list_path = "__files.txt" python_files = "\n".join( determine_python_files(typechecked_sources.source_files.snapshot.files) ) file_list_digest_request = Get( Digest, CreateDigest([FileContent(file_list_path, python_files.encode())]), ) typechecked_venv_pex_request = Get( VenvPex, PexRequest( output_filename="typechecked_venv.pex", internal_only=True, pex_path=[requirements_pex, mypy_extra_requirements_pex], interpreter_constraints=partition.interpreter_constraints, ), ) typechecked_venv_pex, file_list_digest = await MultiGet( typechecked_venv_pex_request, file_list_digest_request ) merged_input_files = await Get( Digest, MergeDigests( [ file_list_digest, plugin_sources.source_files.snapshot.digest, typechecked_srcs_snapshot.digest, typechecked_venv_pex.digest, config_digest, ] ), ) all_used_source_roots = sorted( set(itertools.chain(plugin_sources.source_roots, typechecked_sources.source_roots)) ) env = { "PEX_EXTRA_SYS_PATH": ":".join(all_used_source_roots), } result = await Get( FallibleProcessResult, VenvPexProcess( mypy_pex, argv=generate_argv( mypy, typechecked_venv_pex=typechecked_venv_pex, file_list_path=file_list_path, python_version=python_version, ), input_digest=merged_input_files, extra_env=env, description=f"Run MyPy on {pluralize(len(typechecked_srcs_snapshot.files), 'file')}.", level=LogLevel.DEBUG, ), ) return TypecheckResult.from_fallible_process_result( result, partition_description=str(sorted(str(c) for c in partition.interpreter_constraints)) )
async def twine_upload(request: PublishToPyPiRequest, twine_subsystem: TwineSubsystem) -> PublishProcesses: dists = tuple(artifact.relpath for pkg in request.packages for artifact in pkg.artifacts if artifact.relpath) if twine_subsystem.skip or not dists: return PublishProcesses() # Too verbose to provide feedback as to why some packages were skipped? skip = None if request.field_set.skip_twine.value: skip = f"(by `{request.field_set.skip_twine.alias}` on {request.field_set.address})" elif not request.field_set.repositories.value: # I'd rather have used the opt_out mechanism on the field set, but that gives no hint as to # why the target was not applicable.. skip = f"(no `{request.field_set.repositories.alias}` specified for {request.field_set.address})" if skip: return PublishProcesses([ PublishPackages( names=dists, description=skip, ), ]) twine_pex, packages_digest, config_files = await MultiGet( Get( VenvPex, PexRequest( output_filename="twine.pex", internal_only=True, requirements=twine_subsystem.pex_requirements(), interpreter_constraints=twine_subsystem. interpreter_constraints, main=twine_subsystem.main, ), ), Get(Digest, MergeDigests(pkg.digest for pkg in request.packages)), Get(ConfigFiles, ConfigFilesRequest, twine_subsystem.config_request()), ) input_digest = await Get( Digest, MergeDigests((packages_digest, config_files.snapshot.digest))) pex_proc_requests = [] twine_envs = await MultiGet( Get(Environment, EnvironmentRequest, twine_env_request(repo)) for repo in request.field_set.repositories.value) for repo, env in zip(request.field_set.repositories.value, twine_envs): pex_proc_requests.append( VenvPexProcess( twine_pex, argv=twine_upload_args(twine_subsystem, config_files, repo, dists), input_digest=input_digest, extra_env=twine_env(env, repo), description=repo, )) processes = await MultiGet( Get(Process, VenvPexProcess, request) for request in pex_proc_requests) return PublishProcesses( PublishPackages( names=dists, process=InteractiveProcess.from_process(process), description=process.description, data=PublishOutputData({"repository": process.description}), ) for process in processes)
async def _setup_constraints_repository_pex( request: _ConstraintsRepositoryPexRequest, python_setup: PythonSetup ) -> _ConstraintsRepositoryPex: # NB: it isn't safe to resolve against the whole constraints file if # platforms are in use. See https://github.com/pantsbuild/pants/issues/12222. if not python_setup.resolve_all_constraints or request.platforms: return _ConstraintsRepositoryPex(None) constraints_path = python_setup.requirement_constraints assert constraints_path is not None constraints_file_contents = await Get( DigestContents, PathGlobs( [constraints_path], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `[python].requirement_constraints`", ), ) constraints_file_reqs = set( parse_requirements_file( constraints_file_contents[0].content.decode(), rel_path=constraints_path ) ) # In requirement strings, Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names url_reqs = set() # E.g., 'foobar@ git+https://github.com/foo/bar.git@branch' name_reqs = set() # E.g., foobar>=1.2.3 name_req_projects = set() for req_str in request.requirements.req_strings: req = PipRequirement.parse(req_str) if req.url: url_reqs.add(req) else: name_reqs.add(req) name_req_projects.add(canonicalize_project_name(req.project_name)) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } # Constraints files must only contain name reqs, not URL reqs (those are already # constrained by their very nature). See https://github.com/pypa/pip/issues/8210. unconstrained_projects = name_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( f"The constraints file {constraints_path} does not contain " f"entries for the following requirements: {', '.join(unconstrained_projects)}.\n\n" f"Ignoring `[python_setup].resolve_all_constraints` option." ) return _ConstraintsRepositoryPex(None) # To get a full set of requirements we must add the URL requirements to the # constraints file, since the latter cannot contain URL requirements. # NB: We can only add the URL requirements we know about here, i.e., those that # are transitive deps of the targets in play. There may be others in the repo. # So we may end up creating a few different repository pexes, each with identical # name requirements but different subsets of URL requirements. Fortunately since # all these repository pexes will have identical pinned versions of everything, # this is not a correctness issue, only a performance one. all_constraints = {str(req) for req in (constraints_file_reqs | url_reqs)} repository_pex = await Get( Pex, PexRequest( description=f"Resolving {constraints_path}", output_filename="repository.pex", internal_only=request.internal_only, requirements=PexRequirements( all_constraints, apply_constraints=True, # TODO: See PexRequirements docs. is_all_constraints_resolve=True, ), interpreter_constraints=request.interpreter_constraints, platforms=request.platforms, additional_args=request.additional_lockfile_args, ), ) return _ConstraintsRepositoryPex(repository_pex)
async def package_python_google_cloud_function( field_set: PythonGoogleCloudFunctionFieldSet, lambdex: Lambdex, platform: Platform, union_membership: UnionMembership, ) -> BuiltPackage: if platform.is_macos: logger.warning( "Google Cloud Functions built on macOS may fail to build. If your function uses any" " third-party dependencies without binary wheels (bdist) for Linux available, it will" " fail to build. If this happens, you will either need to update your dependencies to" " only use dependencies with pre-built wheels, or find a Linux environment to run" f" {bin_name()} package. (See https://realpython.com/python-wheels/ for more about" " wheels.)\n\n(If the build does not raise an exception, it's safe to use macOS.)" ) output_filename = field_set.output_path.value_or_default( # Cloud Functions typically use the .zip suffix, so we use that instead of .pex. file_ending="zip", ) # We hardcode the platform value to the appropriate one for each Google Cloud Function runtime. # (Running the "hello world" cloud function in the example code will report the platform, and can be # used to verify correctness of these platform strings.) pex_platforms = [] interpreter_version = field_set.runtime.to_interpreter_version() if interpreter_version: py_major, py_minor = interpreter_version platform_str = f"linux_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}" # set pymalloc ABI flag - this was removed in python 3.8 https://bugs.python.org/issue36707 if py_major <= 3 and py_minor < 8: platform_str += "m" pex_platforms.append(platform_str) additional_pex_args = ( # Ensure we can resolve manylinux wheels in addition to any AMI-specific wheels. "--manylinux=manylinux2014", # When we're executing Pex on Linux, allow a local interpreter to be resolved if # available and matching the AMI platform. "--resolve-local-platforms", ) complete_platforms = await Get(CompletePlatforms, PexCompletePlatformsField, field_set.complete_platforms) pex_request = PexFromTargetsRequest( addresses=[field_set.address], internal_only=False, output_filename=output_filename, platforms=PexPlatforms(pex_platforms), complete_platforms=complete_platforms, additional_args=additional_pex_args, additional_lockfile_args=additional_pex_args, ) lambdex_request = PexRequest( output_filename="lambdex.pex", internal_only=True, requirements=lambdex.pex_requirements(), interpreter_constraints=lambdex.interpreter_constraints, main=lambdex.main, ) lambdex_pex, pex_result, handler, transitive_targets = await MultiGet( Get(VenvPex, PexRequest, lambdex_request), Get(Pex, PexFromTargetsRequest, pex_request), Get(ResolvedPythonGoogleHandler, ResolvePythonGoogleHandlerRequest(field_set.handler)), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) # Warn if users depend on `files` targets, which won't be included in the PEX and is a common # gotcha. file_tgts = targets_with_sources_types([FileSourceField], transitive_targets.dependencies, union_membership) if file_tgts: files_addresses = sorted(tgt.address.spec for tgt in file_tgts) logger.warning( f"The `python_google_cloud_function` target {field_set.address} transitively depends " "on the below `files` targets, but Pants will not include them in the built Cloud " "Function. Filesystem APIs like `open()` are not able to load files within the binary " "itself; instead, they read from the current working directory." f"\n\nInstead, use `resources` targets. See {doc_url('resources')}." f"\n\nFiles targets dependencies: {files_addresses}") # NB: Lambdex modifies its input pex in-place, so the input file is also the output file. result = await Get( ProcessResult, VenvPexProcess( lambdex_pex, argv=("build", "-M", "main.py", "-e", handler.val, output_filename), input_digest=pex_result.digest, output_files=(output_filename, ), description=f"Setting up handler in {output_filename}", ), ) extra_log_data: list[tuple[str, str]] = [] if field_set.runtime.value: extra_log_data.append(("Runtime", field_set.runtime.value)) extra_log_data.extend( ("Complete platform", path) for path in complete_platforms) # The GCP-facing handler function is always main.handler, which is the # wrapper injected by lambdex that manages invocation of the actual handler. extra_log_data.append(("Handler", "main.handler")) first_column_width = 4 + max(len(header) for header, _ in extra_log_data) artifact = BuiltPackageArtifact( output_filename, extra_log_lines=tuple( f"{header.rjust(first_column_width, ' ')}: {data}" for header, data in extra_log_data), ) return BuiltPackage(digest=result.output_digest, artifacts=(artifact, ))
async def flake8_lint_partition(partition: Flake8Partition, flake8: Flake8, lint_subsystem: LintSubsystem) -> LintResult: flake8_pex_request = Get( Pex, PexRequest( output_filename="flake8.pex", internal_only=True, requirements=PexRequirements(flake8.all_requirements), interpreter_constraints=partition.interpreter_constraints, entry_point=flake8.entry_point, ), ) config_digest_request = Get( Digest, PathGlobs( globs=[flake8.config] if flake8.config else [], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin="the option `--flake8-config`", ), ) source_files_request = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)) flake8_pex, config_digest, source_files = await MultiGet( flake8_pex_request, config_digest_request, source_files_request) input_digest = await Get( Digest, MergeDigests( (source_files.snapshot.digest, flake8_pex.digest, config_digest)), ) report_file_name = "flake8_report.txt" if lint_subsystem.reports_dir else None result = await Get( FallibleProcessResult, PexProcess( flake8_pex, argv=generate_args(source_files=source_files, flake8=flake8, report_file_name=report_file_name), input_digest=input_digest, output_files=(report_file_name, ) if report_file_name else None, description= f"Run Flake8 on {pluralize(len(partition.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) report = None if report_file_name: report_digest = await Get( Digest, DigestSubset( result.output_digest, PathGlobs( [report_file_name], glob_match_error_behavior=GlobMatchErrorBehavior.warn, description_of_origin="Flake8 report file", ), ), ) report = LintReport(report_file_name, report_digest) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)), report=report, )
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, complete_env: CompleteEnvironment, ) -> TestSetup: transitive_targets, plugin_setups = await MultiGet( Get(TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])), Get(AllPytestPluginSetups, AllPytestPluginSetupsRequest(request.field_set.address)), ) all_targets = transitive_targets.closure interpreter_constraints = InterpreterConstraints.create_from_targets( all_targets, python_setup) requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( [request.field_set.address], internal_only=True, resolve_and_lockfile=request.field_set.resolve. resolve_and_lockfile(python_setup), ), ) pytest_pex_get = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=pytest.pex_requirements(), interpreter_constraints=interpreter_constraints, internal_only=True, ), ) # Ensure that the empty extra output dir exists. extra_output_directory_digest_get = Get( Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)])) prepared_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_get = Get( SourceFiles, SourceFilesRequest([request.field_set.sources])) ( pytest_pex, requirements_pex, prepared_sources, field_set_source_files, extra_output_directory_digest, ) = await MultiGet( pytest_pex_get, requirements_pex_get, prepared_sources_get, field_set_source_files_get, extra_output_directory_digest_get, ) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [request.field_set.address], interpreter_constraints=interpreter_constraints, sources=prepared_sources, ), ) pytest_runner_pex_get = Get( VenvPex, PexRequest( output_filename="pytest_runner.pex", interpreter_constraints=interpreter_constraints, main=pytest.main, internal_only=True, pex_path=[pytest_pex, requirements_pex, local_dists.pex], ), ) config_files_get = Get( ConfigFiles, ConfigFilesRequest, pytest.config_request(field_set_source_files.snapshot.dirs), ) pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get, config_files_get) input_digest = await Get( Digest, MergeDigests(( coverage_config.digest, local_dists.remaining_sources.source_files.snapshot.digest, config_files.snapshot.digest, extra_output_directory_digest, *(plugin_setup.digest for plugin_setup in plugin_setups), )), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend((f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}")) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: pytest.validate_pytest_cov_included() output_files.append(".coverage") if coverage_subsystem.filter: cov_args = [f"--cov={morf}" for morf in coverage_subsystem.filter] else: # N.B.: Passing `--cov=` or `--cov=.` to communicate "record coverage for all sources" # fails in certain contexts as detailed in: # https://github.com/pantsbuild/pants/issues/12390 # Instead we focus coverage on just the directories containing python source files # materialized to the Process chroot. cov_args = [ f"--cov={source_root}" for source_root in prepared_sources.source_roots ] coverage_args = [ "--cov-report=", # Turn off output. f"--cov-config={coverage_config.path}", *cov_args, ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), **test_extra_env.env, # NOTE: `complete_env` intentionally after `test_extra_env` to allow overriding within # `python_tests` **complete_env.get_subset(request.field_set.extra_env_vars.value or ()), } # Cache test runs only if they are successful, or not at all if `--test-force`. cache_scope = (ProcessCacheScope.PER_SESSION if test_subsystem.force else ProcessCacheScope.SUCCESSFUL) process = await Get( Process, VenvPexProcess( pytest_runner_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_directories=(_EXTRA_OUTPUT_DIR, ), output_files=output_files, timeout_seconds=request.field_set.timeout. calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, cache_scope=cache_scope, ), ) return TestSetup(process, results_file_name=results_file_name)
async def create_python_binary_run_request( field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults, pex_env: PexEnvironment, ) -> RunRequest: entry_point = field_set.entry_point.value if entry_point is None: binary_source_paths = await Get( Paths, PathGlobs, field_set.sources.path_globs(FilesNotFoundBehavior.error)) if len(binary_source_paths.files) != 1: raise InvalidFieldException( "No `entry_point` was set for the target " f"{repr(field_set.address)}, so it must have exactly one source, but it has " f"{len(binary_source_paths.files)}") entry_point_path = binary_source_paths.files[0] source_root = await Get( SourceRoot, SourceRootRequest, SourceRootRequest.for_file(entry_point_path), ) entry_point = PythonBinarySources.translate_source_file_to_entry_point( os.path.relpath(entry_point_path, source_root.path)) transitive_targets = await Get(TransitiveTargets, Addresses([field_set.address])) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in runner_pex_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([field_set.address], internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) output_filename = f"{field_set.address.target_name}.pex" runner_pex_request = Get( Pex, PexRequest( output_filename=output_filename, interpreter_constraints=requirements_pex_request. interpreter_constraints, additional_args=field_set.generate_additional_args( python_binary_defaults), internal_only=True, # Note that the entry point file is not in the Pex itself, but on the # PEX_PATH. This works fine! entry_point=entry_point, ), ) requirements, sources, runner_pex = await MultiGet(requirements_request, sources_request, runner_pex_request) merged_digest = await Get( Digest, MergeDigests([ requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) args = pex_env.create_argv(in_chroot(runner_pex.name), python=runner_pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=runner_pex.python is not None), "PEX_PATH": in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_pex_binary_run_request( field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment, ) -> RunRequest: entry_point, transitive_targets = await MultiGet( Get( ResolvedPexEntryPoint, ResolvePexEntryPointRequest(field_set.entry_point, field_set.sources), ), Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), ) transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([field_set.address])) # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) # so that we can get the interpreter constraints for use in runner_pex_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([field_set.address], internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)) output_filename = f"{field_set.address.target_name}.pex" runner_pex_request = Get( Pex, PexRequest( output_filename=output_filename, interpreter_constraints=requirements_pex_request. interpreter_constraints, additional_args=field_set.generate_additional_args( pex_binary_defaults), internal_only=True, # Note that the entry point file is not in the PEX itself. It's loaded by setting # `PEX_EXTRA_SYS_PATH`. entry_point=entry_point.val, ), ) requirements, sources, runner_pex = await MultiGet(requirements_request, sources_request, runner_pex_request) merged_digest = await Get( Digest, MergeDigests([ requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest ]), ) def in_chroot(relpath: str) -> str: return os.path.join("{chroot}", relpath) args = pex_env.create_argv(in_chroot(runner_pex.name), python=runner_pex.python) chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] extra_env = { **pex_env.environment_dict(python_configured=runner_pex.python is not None), "PEX_PATH": in_chroot(requirements_pex_request.output_filename), "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def generate_python_from_protobuf( request: GeneratePythonFromProtobufRequest, protoc: Protoc, grpc_python_plugin: GrpcPythonPlugin, python_protobuf_subsystem: PythonProtobufSubsystem, python_protobuf_mypy_plugin: PythonProtobufMypyPlugin, ) -> GeneratedSources: download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)) output_dir = "_generated_files" create_output_dir_request = Get(Digest, CreateDigest([Directory(output_dir)])) # Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't # actually generate those dependencies; it only needs to look at their .proto files to work # with imports. transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.protocol_target.address])) # NB: By stripping the source roots, we avoid having to set the value `--proto_path` # for Protobuf imports to be discoverable. all_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest( (tgt.get(Sources) for tgt in transitive_targets.closure), for_sources_types=(ProtobufSources, ), ), ) target_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest([request.protocol_target[ProtobufSources]])) ( downloaded_protoc_binary, empty_output_dir, all_sources_stripped, target_sources_stripped, ) = await MultiGet( download_protoc_request, create_output_dir_request, all_stripped_sources_request, target_stripped_sources_request, ) protoc_gen_mypy_script = "protoc-gen-mypy" protoc_gen_mypy_grpc_script = "protoc-gen-mypy_grpc" mypy_pex = None mypy_request = PexRequest( output_filename="mypy_protobuf.pex", internal_only=True, requirements=PexRequirements([ python_protobuf_mypy_plugin.plugin_requirement( python_protobuf_subsystem) ]), interpreter_constraints=PexInterpreterConstraints( python_protobuf_mypy_plugin.interpreter_constraints), ) if python_protobuf_subsystem.mypy_plugin: mypy_pex = await Get( VenvPex, VenvPexRequest( bin_names=[protoc_gen_mypy_script], pex_request=mypy_request, ), ) if request.protocol_target.get(ProtobufGrpcToggle).value: mypy_info = await Get(PexResolveInfo, VenvPex, mypy_pex) # In order to generate stubs for gRPC code, we need mypy-protobuf 2.0 or above. if any(dist_info.project_name == "mypy-protobuf" and dist_info.version.major >= 2 for dist_info in mypy_info): # TODO: Use `pex_path` once VenvPex stores a Pex field. mypy_pex = await Get( VenvPex, VenvPexRequest( bin_names=[ protoc_gen_mypy_script, protoc_gen_mypy_grpc_script ], pex_request=mypy_request, ), ) downloaded_grpc_plugin = (await Get( DownloadedExternalTool, ExternalToolRequest, grpc_python_plugin.get_request(Platform.current), ) if request.protocol_target.get(ProtobufGrpcToggle).value else None) unmerged_digests = [ all_sources_stripped.snapshot.digest, downloaded_protoc_binary.digest, empty_output_dir, ] if mypy_pex: unmerged_digests.append(mypy_pex.digest) if downloaded_grpc_plugin: unmerged_digests.append(downloaded_grpc_plugin.digest) input_digest = await Get(Digest, MergeDigests(unmerged_digests)) argv = [downloaded_protoc_binary.exe, "--python_out", output_dir] if mypy_pex: argv.extend([ f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}", "--mypy_out", output_dir, ]) if downloaded_grpc_plugin: argv.extend([ f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}", "--grpc_out", output_dir ]) if mypy_pex and protoc_gen_mypy_grpc_script in mypy_pex.bin: argv.extend([ f"--plugin=protoc-gen-mypy_grpc={mypy_pex.bin[protoc_gen_mypy_grpc_script].argv0}", "--mypy_grpc_out", output_dir, ]) argv.extend(target_sources_stripped.snapshot.files) result = await Get( ProcessResult, Process( argv, input_digest=input_digest, description= f"Generating Python sources from {request.protocol_target.address}.", level=LogLevel.DEBUG, output_directories=(output_dir, ), ), ) # We must do some path manipulation on the output digest for it to look like normal sources, # including adding back a source root. py_source_root = request.protocol_target.get(PythonSourceRootField).value if py_source_root: # Verify that the python source root specified by the target is in fact a source root. source_root_request = SourceRootRequest(PurePath(py_source_root)) else: # The target didn't specify a python source root, so use the protobuf_library's source root. source_root_request = SourceRootRequest.for_target( request.protocol_target) normalized_digest, source_root = await MultiGet( Get(Digest, RemovePrefix(result.output_digest, output_dir)), Get(SourceRoot, SourceRootRequest, source_root_request), ) source_root_restored = (await Get( Snapshot, AddPrefix(normalized_digest, source_root.path)) if source_root.path != "." else await Get( Snapshot, Digest, normalized_digest)) return GeneratedSources(source_root_restored)
async def bandit_lint_partition(partition: BanditPartition, bandit: Bandit, lint_subsystem: LintSubsystem) -> LintResult: bandit_pex_get = Get( VenvPex, PexRequest( output_filename="bandit.pex", internal_only=True, requirements=PexRequirements(bandit.all_requirements), interpreter_constraints=partition.interpreter_constraints, main=bandit.main, ), ) config_files_get = Get(ConfigFiles, ConfigFilesRequest, bandit.config_request) source_files_get = Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in partition.field_sets)) bandit_pex, config_files, source_files = await MultiGet( bandit_pex_get, config_files_get, source_files_get) input_digest = await Get( Digest, MergeDigests( (source_files.snapshot.digest, config_files.snapshot.digest))) report_file_name = "bandit_report.txt" if lint_subsystem.reports_dir else None result = await Get( FallibleProcessResult, VenvPexProcess( bandit_pex, argv=generate_argv(source_files, bandit, report_file_name=report_file_name), input_digest=input_digest, description= f"Run Bandit on {pluralize(len(partition.field_sets), 'file')}.", output_files=(report_file_name, ) if report_file_name else None, level=LogLevel.DEBUG, ), ) report = None if report_file_name: report_digest = await Get( Digest, DigestSubset( result.output_digest, PathGlobs( [report_file_name], glob_match_error_behavior=GlobMatchErrorBehavior.warn, description_of_origin="Bandit report file", ), ), ) report = LintReport(report_file_name, report_digest) return LintResult.from_fallible_process_result( result, partition_description=str( sorted(str(c) for c in partition.interpreter_constraints)), report=report, )
async def create_ipython_repl_request(repl: IPythonRepl, ipython: IPython, pex_env: PexEnvironment) -> ReplRequest: # Note that we get an intermediate PexRequest here (instead of going straight to a Pex) so # that we can get the interpreter constraints for use in ipython_request/local_dists_request. requirements_pex_request = await Get( PexRequest, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements( (tgt.address for tgt in repl.targets), internal_only=True), ) requirements_request = Get(Pex, PexRequest, requirements_pex_request) sources_request = Get( PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)) ipython_request = Get( Pex, PexRequest( output_filename="ipython.pex", main=ipython.main, requirements=ipython.pex_requirements(), interpreter_constraints=requirements_pex_request. interpreter_constraints, internal_only=True, ), ) requirements_pex, sources, ipython_pex = await MultiGet( requirements_request, sources_request, ipython_request) local_dists = await Get( LocalDistsPex, LocalDistsPexRequest( [tgt.address for tgt in repl.targets], internal_only=True, interpreter_constraints=requirements_pex_request. interpreter_constraints, sources=sources, ), ) merged_digest = await Get( Digest, MergeDigests(( requirements_pex.digest, local_dists.pex.digest, local_dists.remaining_sources.source_files.snapshot.digest, ipython_pex.digest, )), ) complete_pex_env = pex_env.in_workspace() args = list( complete_pex_env.create_argv(repl.in_chroot(ipython_pex.name), python=ipython_pex.python)) if ipython.options.ignore_cwd: args.append("--ignore-cwd") chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots] extra_env = { **complete_pex_env.environment_dict(python_configured=ipython_pex.python is not None), "PEX_PATH": os.pathsep.join([ repl.in_chroot(requirements_pex_request.output_filename), repl.in_chroot(local_dists.pex.name), ]), "PEX_EXTRA_SYS_PATH": os.pathsep.join(chrooted_source_roots), } return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def generate_python_from_protobuf( request: GeneratePythonFromProtobufRequest, protoc: Protoc, grpc_python_plugin: GrpcPythonPlugin, python_protobuf_subsystem: PythonProtobufSubsystem, ) -> GeneratedSources: download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)) output_dir = "_generated_files" create_output_dir_request = Get(Digest, CreateDigest([Directory(output_dir)])) # Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't # actually generate those dependencies; it only needs to look at their .proto files to work # with imports. # TODO(#10917): Use TransitiveTargets instead of TransitiveTargetsLite. transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequestLite([request.protocol_target.address])) # NB: By stripping the source roots, we avoid having to set the value `--proto_path` # for Protobuf imports to be discoverable. all_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest( (tgt.get(Sources) for tgt in transitive_targets.closure), for_sources_types=(ProtobufSources, ), ), ) target_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest([request.protocol_target[ProtobufSources]])) ( downloaded_protoc_binary, empty_output_dir, all_sources_stripped, target_sources_stripped, ) = await MultiGet( download_protoc_request, create_output_dir_request, all_stripped_sources_request, target_stripped_sources_request, ) # To run the MyPy Protobuf plugin, we first install it with Pex, then extract the wheels and # point Protoc to the extracted wheels with its `--plugin` argument. extracted_mypy_wheels = None if python_protobuf_subsystem.mypy_plugin: mypy_pex = await Get( Pex, PexRequest( output_filename="mypy_protobuf.pex", internal_only=True, requirements=PexRequirements( [python_protobuf_subsystem.mypy_plugin_version]), # This is solely to ensure that we use an appropriate interpreter when resolving # the distribution. We don't actually run the distribution directly with Python, # as we extract out its binary. interpreter_constraints=PexInterpreterConstraints( ["CPython>=3.5"]), ), ) extracted_mypy_wheels = await Get(ExtractedPexDistributions, Pex, mypy_pex) downloaded_grpc_plugin = (await Get( DownloadedExternalTool, ExternalToolRequest, grpc_python_plugin.get_request(Platform.current), ) if request.protocol_target.get(ProtobufGrcpToggle).value else None) unmerged_digests = [ all_sources_stripped.snapshot.digest, downloaded_protoc_binary.digest, empty_output_dir, ] if extracted_mypy_wheels: unmerged_digests.append(extracted_mypy_wheels.digest) if downloaded_grpc_plugin: unmerged_digests.append(downloaded_grpc_plugin.digest) input_digest = await Get(Digest, MergeDigests(unmerged_digests)) argv = [downloaded_protoc_binary.exe, "--python_out", output_dir] if extracted_mypy_wheels: mypy_plugin_path = next( p for p in extracted_mypy_wheels.wheel_directory_paths if p.startswith(".deps/mypy_protobuf-")) argv.extend([ f"--plugin=protoc-gen-mypy={mypy_plugin_path}/bin/protoc-gen-mypy", "--mypy_out", output_dir, ]) if downloaded_grpc_plugin: argv.extend([ f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}", "--grpc_out", output_dir ]) argv.extend(target_sources_stripped.snapshot.files) env = {} if extracted_mypy_wheels: env["PYTHONPATH"] = ":".join( extracted_mypy_wheels.wheel_directory_paths) result = await Get( ProcessResult, Process( argv, env=env, input_digest=input_digest, description= f"Generating Python sources from {request.protocol_target.address}.", level=LogLevel.DEBUG, output_directories=(output_dir, ), ), ) # We must do some path manipulation on the output digest for it to look like normal sources, # including adding back a source root. py_source_root = request.protocol_target.get(PythonSourceRootField).value if py_source_root: # Verify that the python source root specified by the target is in fact a source root. source_root_request = SourceRootRequest(PurePath(py_source_root)) else: # The target didn't specify a python source root, so use the protobuf_library's source root. source_root_request = SourceRootRequest.for_target( request.protocol_target) normalized_digest, source_root = await MultiGet( Get(Digest, RemovePrefix(result.output_digest, output_dir)), Get(SourceRoot, SourceRootRequest, source_root_request), ) source_root_restored = (await Get( Snapshot, AddPrefix(normalized_digest, source_root.path)) if source_root.path != "." else await Get( Snapshot, Digest, normalized_digest)) return GeneratedSources(source_root_restored)
async def run_pep517_build(request: DistBuildRequest, python_setup: PythonSetup) -> DistBuildResult: # Note that this pex has no entrypoint. We use it to run our generated shim, which # in turn imports from and invokes the build backend. build_backend_pex = await Get( VenvPex, PexRequest( output_filename="build_backend.pex", internal_only=True, requirements=request.build_system.requires, pex_path=request.extra_build_time_requirements, interpreter_constraints=request.interpreter_constraints, ), ) dist_dir = "dist" backend_shim_name = "backend_shim.py" backend_shim_path = os.path.join(request.working_directory, backend_shim_name) backend_shim_digest = await Get( Digest, CreateDigest([ FileContent(backend_shim_path, interpolate_backend_shim(dist_dir, request)), ]), ) merged_digest = await Get( Digest, MergeDigests((request.input, backend_shim_digest))) extra_env = { **(request.extra_build_time_env or {}), "PEX_EXTRA_SYS_PATH": os.pathsep.join(request.build_time_source_roots), } if python_setup.macos_big_sur_compatibility and is_macos_big_sur(): extra_env["MACOSX_DEPLOYMENT_TARGET"] = "10.16" result = await Get( ProcessResult, VenvPexProcess( build_backend_pex, argv=(backend_shim_name, ), input_digest=merged_digest, extra_env=extra_env, working_directory=request.working_directory, output_directories=( dist_dir, ), # Relative to the working_directory. description= (f"Run {request.build_system.build_backend} for {request.target_address_spec}" if request.target_address_spec else f"Run {request.build_system.build_backend}"), level=LogLevel.DEBUG, ), ) output_lines = result.stdout.decode().splitlines() paths = {} for line in output_lines: for dist_type in ["wheel", "sdist"]: if line.startswith(f"{dist_type}: "): paths[dist_type] = line[len(dist_type) + 2:].strip() # Note that output_digest paths are relative to the working_directory. output_digest = await Get(Digest, RemovePrefix(result.output_digest, dist_dir)) output_snapshot = await Get(Snapshot, Digest, output_digest) for dist_type, path in paths.items(): if path not in output_snapshot.files: raise BuildBackendError( softwrap(f""" Build backend {request.build_system.build_backend} did not create expected {dist_type} file {path} """)) return DistBuildResult(output_digest, wheel_path=paths.get("wheel"), sdist_path=paths.get("sdist"))
async def setup_pytest_for_target( request: TestSetupRequest, pytest: PyTest, test_subsystem: TestSubsystem, python_setup: PythonSetup, coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, ) -> TestSetup: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest([request.field_set.address])) all_targets = transitive_targets.closure interpreter_constraints = PexInterpreterConstraints.create_from_targets( all_targets, python_setup) requirements_pex_get = Get( Pex, PexFromTargetsRequest, PexFromTargetsRequest.for_requirements([request.field_set.address], internal_only=True), ) pytest_pex_get = Get( Pex, PexRequest( output_filename="pytest.pex", requirements=PexRequirements(pytest.get_requirement_strings()), interpreter_constraints=interpreter_constraints, internal_only=True, ), ) extra_output_directory_digest_get = Get( Digest, CreateDigest([Directory(_EXTRA_OUTPUT_DIR)])) prepared_sources_get = Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True)) build_package_dependencies_get = Get( BuiltPackageDependencies, BuildPackageDependenciesRequest( request.field_set.runtime_package_dependencies), ) # Get the file names for the test_target so that we can specify to Pytest precisely which files # to test, rather than using auto-discovery. field_set_source_files_get = Get( SourceFiles, SourceFilesRequest([request.field_set.sources])) ( pytest_pex, requirements_pex, prepared_sources, field_set_source_files, built_package_dependencies, extra_output_directory_digest, ) = await MultiGet( pytest_pex_get, requirements_pex_get, prepared_sources_get, field_set_source_files_get, build_package_dependencies_get, extra_output_directory_digest_get, ) pytest_runner_pex_get = Get( VenvPex, PexRequest( output_filename="pytest_runner.pex", interpreter_constraints=interpreter_constraints, main=ConsoleScript("pytest"), internal_only=True, pex_path=[pytest_pex, requirements_pex], ), ) config_files_get = Get( ConfigFiles, ConfigFilesRequest, pytest.config_request(field_set_source_files.snapshot.dirs), ) pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get, config_files_get) input_digest = await Get( Digest, MergeDigests(( coverage_config.digest, prepared_sources.source_files.snapshot.digest, config_files.snapshot.digest, extra_output_directory_digest, *(pkg.digest for pkg in built_package_dependencies), )), ) add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"] output_files = [] results_file_name = None if pytest.options.junit_xml_dir and not request.is_debug: results_file_name = f"{request.field_set.address.path_safe_spec}.xml" add_opts.extend((f"--junitxml={results_file_name}", "-o", f"junit_family={pytest.options.junit_family}")) output_files.append(results_file_name) coverage_args = [] if test_subsystem.use_coverage and not request.is_debug: output_files.append(".coverage") cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else ( ".", ) coverage_args = [ "--cov-report=", # Turn off output. f"--cov-config={coverage_config.path}", *itertools.chain.from_iterable(["--cov", cov_path] for cov_path in cov_paths), ] extra_env = { "PYTEST_ADDOPTS": " ".join(add_opts), "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), **test_extra_env.env, } # Cache test runs only if they are successful, or not at all if `--test-force`. cache_scope = ProcessCacheScope.NEVER if test_subsystem.force else ProcessCacheScope.SUCCESSFUL process = await Get( Process, VenvPexProcess( pytest_runner_pex, argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files), extra_env=extra_env, input_digest=input_digest, output_directories=(_EXTRA_OUTPUT_DIR, ), output_files=output_files, timeout_seconds=request.field_set.timeout. calculate_from_global_options(pytest), execution_slot_variable=pytest.options.execution_slot_var, description=f"Run Pytest for {request.field_set.address}", level=LogLevel.DEBUG, cache_scope=cache_scope, ), ) return TestSetup(process, results_file_name=results_file_name)
async def collect_fixture_configs( _request: CollectFixtureConfigsRequest, pytest: PyTest, python_setup: PythonSetup, test_extra_env: TestExtraEnv, targets: Targets, ) -> CollectedJVMLockfileFixtureConfigs: addresses = [tgt.address for tgt in targets] transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(addresses)) all_targets = transitive_targets.closure interpreter_constraints = InterpreterConstraints.create_from_targets( all_targets, python_setup) pytest_pex, requirements_pex, prepared_sources, root_sources = await MultiGet( Get( Pex, PexRequest( output_filename="pytest.pex", requirements=pytest.pex_requirements(), interpreter_constraints=interpreter_constraints, internal_only=True, ), ), Get(Pex, RequirementsPexRequest(addresses)), Get( PythonSourceFiles, PythonSourceFilesRequest(all_targets, include_files=True, include_resources=True), ), Get( PythonSourceFiles, PythonSourceFilesRequest(targets), ), ) script_content = FileContent(path="collect-fixtures.py", content=COLLECTION_SCRIPT.encode(), is_executable=True) script_digest = await Get(Digest, CreateDigest([script_content])) pytest_runner_pex_get = Get( VenvPex, PexRequest( output_filename="pytest_runner.pex", interpreter_constraints=interpreter_constraints, main=EntryPoint(PurePath(script_content.path).stem), sources=script_digest, internal_only=True, pex_path=[ pytest_pex, requirements_pex, ], ), ) config_file_dirs = list( group_by_dir(prepared_sources.source_files.files).keys()) config_files_get = Get( ConfigFiles, ConfigFilesRequest, pytest.config_request(config_file_dirs), ) pytest_runner_pex, config_files = await MultiGet(pytest_runner_pex_get, config_files_get) pytest_config_digest = config_files.snapshot.digest input_digest = await Get( Digest, MergeDigests(( prepared_sources.source_files.snapshot.digest, pytest_config_digest, )), ) extra_env = { "PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots), **test_extra_env.env, } process = await Get( Process, VenvPexProcess( pytest_runner_pex, argv=[ name for name in root_sources.source_files.files if name.endswith(".py") ], extra_env=extra_env, input_digest=input_digest, output_files=("tests.json", ), description="Collect test lockfile requirements from all tests.", level=LogLevel.DEBUG, cache_scope=ProcessCacheScope.PER_SESSION, ), ) result = await Get(ProcessResult, Process, process) digest_contents = await Get(DigestContents, Digest, result.output_digest) assert len(digest_contents) == 1 assert digest_contents[0].path == "tests.json" raw_config_data = json.loads(digest_contents[0].content) configs = [] for item in raw_config_data: config = JVMLockfileFixtureConfig( definition=JVMLockfileFixtureDefinition.from_kwargs( item["kwargs"]), test_file_path=item["test_file_path"], ) configs.append(config) return CollectedJVMLockfileFixtureConfigs(configs)
async def pex_from_targets(request: PexFromTargetsRequest, python_setup: PythonSetup) -> PexRequest: if request.direct_deps_only: targets = await Get(Targets, Addresses(request.addresses)) direct_deps = await MultiGet( Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in targets) all_targets = FrozenOrderedSet(itertools.chain(*direct_deps, targets)) else: transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest(request.addresses)) all_targets = transitive_targets.closure input_digests = [] if request.additional_sources: input_digests.append(request.additional_sources) if request.include_source_files: prepared_sources = await Get(StrippedPythonSourceFiles, PythonSourceFilesRequest(all_targets)) input_digests.append( prepared_sources.stripped_source_files.snapshot.digest) merged_input_digest = await Get(Digest, MergeDigests(input_digests)) if request.hardcoded_interpreter_constraints: interpreter_constraints = request.hardcoded_interpreter_constraints else: calculated_constraints = PexInterpreterConstraints.create_from_compatibility_fields( (tgt[PythonInterpreterCompatibility] for tgt in all_targets if tgt.has_field(PythonInterpreterCompatibility)), python_setup, ) # If there are no targets, we fall back to the global constraints. This is relevant, # for example, when running `./pants repl` with no specs. interpreter_constraints = calculated_constraints or PexInterpreterConstraints( python_setup.interpreter_constraints) exact_reqs = PexRequirements.create_from_requirement_fields( (tgt[PythonRequirementsField] for tgt in all_targets if tgt.has_field(PythonRequirementsField)), additional_requirements=request.additional_requirements, ) requirements = exact_reqs description = request.description if python_setup.requirement_constraints: # In requirement strings Foo_-Bar.BAZ and foo-bar-baz refer to the same project. We let # packaging canonicalize for us. # See: https://www.python.org/dev/peps/pep-0503/#normalized-names exact_req_projects = { canonicalize_project_name(Requirement.parse(req).project_name) for req in exact_reqs } constraints_file_contents = await Get( DigestContents, PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin= "the option `--python-setup-requirement-constraints`", ), ) constraints_file_reqs = set( parse_requirements( next(iter(constraints_file_contents)).content.decode())) constraint_file_projects = { canonicalize_project_name(req.project_name) for req in constraints_file_reqs } unconstrained_projects = exact_req_projects - constraint_file_projects if unconstrained_projects: logger.warning( f"The constraints file {python_setup.requirement_constraints} does not contain " f"entries for the following requirements: {', '.join(unconstrained_projects)}" ) if python_setup.resolve_all_constraints == ResolveAllConstraintsOption.ALWAYS or ( python_setup.resolve_all_constraints == ResolveAllConstraintsOption.NONDEPLOYABLES and request.internal_only): if unconstrained_projects: logger.warning( "Ignoring resolve_all_constraints setting in [python_setup] scope " "because constraints file does not cover all requirements." ) else: requirements = PexRequirements( str(req) for req in constraints_file_reqs) description = description or f"Resolving {python_setup.requirement_constraints}" elif (python_setup.resolve_all_constraints != ResolveAllConstraintsOption.NEVER and python_setup.resolve_all_constraints_was_set_explicitly()): raise ValueError( f"[python-setup].resolve_all_constraints is set to " f"{python_setup.resolve_all_constraints.value}, so " f"[python-setup].requirement_constraints must also be provided.") return PexRequest( output_filename=request.output_filename, internal_only=request.internal_only, requirements=requirements, interpreter_constraints=interpreter_constraints, platforms=request.platforms, entry_point=request.entry_point, sources=merged_input_digest, additional_inputs=request.additional_inputs, additional_args=request.additional_args, description=description, )
async def run_setup_py(req: RunSetupPyRequest, setuptools: Setuptools, python_setup: PythonSetup) -> RunSetupPyResult: """Run a setup.py command on a single exported target.""" # Note that this pex has no entrypoint. We use it to run our generated setup.py, which # in turn imports from and invokes setuptools. setuptools_pex = await Get( VenvPex, PexRequest( output_filename="setuptools.pex", internal_only=True, requirements=setuptools.pex_requirements(), interpreter_constraints=req.interpreter_constraints, ), ) # We prefix the entire chroot, and run with this prefix as the cwd, so that we can capture any # changes setup made within it (e.g., when running 'develop') without also capturing other # artifacts of the pex process invocation. chroot_prefix = "chroot" # The setuptools dist dir, created by it under the chroot (not to be confused with # pants's own dist dir, at the buildroot). dist_dir = "dist" prefixed_chroot = await Get(Digest, AddPrefix(req.chroot.digest, chroot_prefix)) # setup.py basically always expects to be run with the cwd as its own directory # (e.g., paths in it are relative to that directory). This is true of the setup.py # we generate and is overwhelmingly likely to be true for existing setup.py files, # as there is no robust way to run them otherwise. setup_script_reldir, setup_script_name = os.path.split( req.chroot.setup_script) working_directory = os.path.join(chroot_prefix, setup_script_reldir) if python_setup.macos_big_sur_compatibility and is_macos_big_sur(): extra_env = {"MACOSX_DEPLOYMENT_TARGET": "10.16"} else: extra_env = {} result = await Get( ProcessResult, VenvPexProcess( setuptools_pex, argv=(setup_script_name, *req.args), input_digest=prefixed_chroot, extra_env=extra_env, working_directory=working_directory, # setuptools commands that create dists write them to the distdir. # TODO: Could there be other useful files to capture? output_directories=( dist_dir, ), # Relative to the working_directory. description= f"Run setuptools for {req.exported_target.target.address}", level=LogLevel.DEBUG, ), ) # Note that output_digest paths are relative to the working_directory. output_digest = await Get(Digest, RemovePrefix(result.output_digest, dist_dir)) return RunSetupPyResult(output_digest)