def test_cache_scope_per_restart() -> None: success_argv = ("/bin/bash", "-c", "echo $RANDOM") failure_argv = ("/bin/bash", "-c", "echo $RANDOM; exit 1") always_cache_success = Process( success_argv, cache_scope=ProcessCacheScope.PER_RESTART_ALWAYS, description="foo") always_cache_failure = Process( failure_argv, cache_scope=ProcessCacheScope.PER_RESTART_ALWAYS, description="foo") success_cache_success = Process( success_argv, cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL, description="foo") success_cache_failure = Process( failure_argv, cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL, description="foo") runner_one = new_rule_runner() def run1(process: Process) -> FallibleProcessResult: return runner_one.request(FallibleProcessResult, [process]) always_cache_success_res1 = run1(always_cache_success) always_cache_failure_res1 = run1(always_cache_failure) success_cache_success_res1 = run1(success_cache_success) success_cache_failure_res1 = run1(success_cache_failure) runner_one.new_session("new session") always_cache_success_res2 = run1(always_cache_success) always_cache_failure_res2 = run1(always_cache_failure) success_cache_success_res2 = run1(success_cache_success) success_cache_failure_res2 = run1(success_cache_failure) # Even with a new session, most results should be memoized. assert always_cache_success_res1 is always_cache_success_res2 assert always_cache_failure_res1 is always_cache_failure_res2 assert success_cache_success_res1 is success_cache_success_res2 assert success_cache_failure_res1 != success_cache_failure_res2 # But a new scheduler removes all memoization. We do not cache to disk. runner_two = new_rule_runner() def run2(process: Process) -> FallibleProcessResult: return runner_two.request(FallibleProcessResult, [process]) assert run2(always_cache_success) != always_cache_success_res1 assert run2(always_cache_failure) != always_cache_failure_res1 assert run2(success_cache_success) != success_cache_success_res1 assert run2(success_cache_failure) != success_cache_failure_res1
async def jvm_process(bash: BashBinary, request: JvmProcess) -> Process: jdk = request.jdk immutable_input_digests = { **jdk.immutable_input_digests, **request.extra_immutable_input_digests, } env = { "PANTS_INTERNAL_ABSOLUTE_PREFIX": "", **jdk.env, **request.extra_env, } use_nailgun = [] if request.use_nailgun: use_nailgun = [ *jdk.immutable_input_digests, *request.extra_nailgun_keys ] return Process( [*jdk.args(bash, request.classpath_entries), *request.argv], input_digest=request.input_digest, immutable_input_digests=immutable_input_digests, use_nailgun=use_nailgun, description=request.description, level=request.level, output_directories=request.output_directories, env=env, platform=request.platform, timeout_seconds=request.timeout_seconds, append_only_caches=jdk.append_only_caches, output_files=request.output_files, cache_scope=request.cache_scope or ProcessCacheScope.SUCCESSFUL, )
def test_pex_execution(rule_runner: RuleRunner) -> None: sources = rule_runner.request( Digest, [ CreateDigest(( FileContent("main.py", b'print("from main")'), FileContent("subdir/sub.py", b'print("from sub")'), )), ], ) pex_output = create_pex_and_get_all_data(rule_runner, main=EntryPoint("main"), sources=sources) pex_files = pex_output["files"] assert "pex" not in pex_files assert "main.py" in pex_files assert "subdir/sub.py" in pex_files # This should run the Pex using the same interpreter used to create it. We must set the `PATH` so that the shebang # works. process = Process( argv=("./test.pex", ), env={"PATH": os.getenv("PATH", "")}, input_digest=pex_output["pex"].digest, description="Run the pex and make sure it works", ) result = rule_runner.request(ProcessResult, [process]) assert result.stdout == b"from main\n"
def test_fallible_failing_command_returns_exited_result(self): request = Process(argv=("/bin/bash", "-c", "exit 1"), description="one-cat") result = self.request(FallibleProcessResult, [request]) self.assertEqual(result.exit_code, 1)
async def a_rule() -> TrueResult: proc = Process( ["/bin/sh", "-c", "true"], description="always true", ) _ = await Get(ProcessResult, MultiPlatformProcess({None: proc})) return TrueResult()
def test_context_object(self): scheduler = self.scheduler def callback(**kwargs) -> None: context = kwargs["context"] assert isinstance(context, StreamingWorkunitContext) completed_workunits = kwargs["completed_workunits"] for workunit in completed_workunits: if "artifacts" in workunit and "stdout_digest" in workunit["artifacts"]: digest = workunit["artifacts"]["stdout_digest"] output = context.single_file_digests_to_bytes([digest]) assert output == (b"stdout output\n",) handler = StreamingWorkunitHandler( scheduler, callbacks=[callback], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.INFO, ) stdout_process = Process( argv=("/bin/bash", "-c", "/bin/echo 'stdout output'"), description="Stdout process" ) with handler.session(): self.request(ProcessResult, [stdout_process])
async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment) -> Process: pex = request.pex complete_pex_env = pex_environment.in_sandbox( working_directory=request.working_directory) argv = complete_pex_env.create_argv(pex.name, *request.argv, python=pex.python) env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), **(request.extra_env or {}), } input_digest = (await Get( Digest, MergeDigests( (pex.digest, request.input_digest))) if request.input_digest else pex.digest) return Process( argv, description=request.description, level=request.level, input_digest=input_digest, working_directory=request.working_directory, env=env, output_files=request.output_files, output_directories=request.output_directories, append_only_caches=complete_pex_env.append_only_caches, timeout_seconds=request.timeout_seconds, execution_slot_variable=request.execution_slot_variable, cache_scope=request.cache_scope, )
def test_output_digest(rule_runner: RuleRunner, working_directory) -> None: # Test that the output files are relative to the working directory, both in how # they're specified, and their paths in the output_digest. input_digest = (rule_runner.request( Digest, [CreateDigest([Directory(working_directory)])], ) if working_directory else EMPTY_DIGEST) process = Process( input_digest=input_digest, argv=("/bin/bash", "-c", "echo -n 'European Burmese' > roland"), description="echo roland", output_files=("roland", ), working_directory=working_directory, ) result = rule_runner.request(ProcessResult, [process]) assert result.output_digest == Digest( fingerprint= "63949aa823baf765eff07b946050d76ec0033144c785a94d3ebd82baa931cd16", serialized_bytes_length=80, ) digest_contents = rule_runner.request(DigestContents, [result.output_digest]) assert digest_contents == DigestContents( [FileContent("roland", b"European Burmese", False)])
def test_pex_execution(rule_runner: RuleRunner) -> None: sources = rule_runner.request( Digest, [ CreateDigest(( FileContent("main.py", b'print("from main")'), FileContent("subdir/sub.py", b'print("from sub")'), )), ], ) pex_output = create_pex_and_get_all_data(rule_runner, entry_point="main", sources=sources) pex_files = pex_output["files"] assert "pex" not in pex_files assert "main.py" in pex_files assert "subdir/sub.py" in pex_files # We reasonably expect there to be a python interpreter on the test-running process's path. env = {"PATH": os.getenv("PATH", "")} process = Process( argv=("python", "test.pex"), env=env, input_digest=pex_output["pex"].digest, description="Run the pex and make sure it works", ) result = rule_runner.request(ProcessResult, [process]) assert result.stdout == b"from main\n"
async def render_classpath_entry( classpath_entry: ClasspathEntry, unzip_binary: UnzipBinary) -> RenderedClasspath: dest_dir = "dest" process_results = await MultiGet( Get( ProcessResult, Process( argv=[ unzip_binary.path, "-d", dest_dir, filename, ], input_digest=classpath_entry.digest, output_directories=(dest_dir, ), description=f"Extract {filename}", ), ) for filename in classpath_entry.filenames) listing_snapshots = await MultiGet( Get(Snapshot, RemovePrefix(pr.output_digest, dest_dir)) for pr in process_results) return RenderedClasspath({ path: set(listing.files) for path, listing in zip(classpath_entry.filenames, listing_snapshots) })
async def parse_python_imports(request: ParsePythonImportsRequest) -> ParsedPythonImports: script = _SCRIPT_FORMAT.format(min_dots=request.string_imports_min_dots).encode() python_interpreter, script_digest, stripped_sources = await MultiGet( Get(PythonExecutable, InterpreterConstraints, request.interpreter_constraints), Get(Digest, CreateDigest([FileContent("__parse_python_imports.py", script)])), Get(StrippedSourceFiles, SourceFilesRequest([request.source])), ) # We operate on PythonSourceField, which should be one file. assert len(stripped_sources.snapshot.files) == 1 file = stripped_sources.snapshot.files[0] input_digest = await Get( Digest, MergeDigests([script_digest, stripped_sources.snapshot.digest]) ) process_result = await Get( ProcessResult, Process( argv=[ python_interpreter.path, "./__parse_python_imports.py", file, ], input_digest=input_digest, description=f"Determine Python imports for {request.source.address}", env={"STRING_IMPORTS": "y" if request.string_imports else "n"}, level=LogLevel.DEBUG, ), ) # See above for where we explicitly encoded as utf8. Even though utf8 is the # default for decode(), we make that explicit here for emphasis. return ParsedPythonImports(process_result.stdout.decode("utf8").strip().splitlines())
async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment) -> Process: argv = pex_environment.create_argv( f"./{request.pex.name}", *request.argv, python=request.pex.python, ) env = { **pex_environment.environment_dict(python_configured=request.pex.python is not None), **(request.extra_env or {}), } process = Process( argv, description=request.description, level=request.level, input_digest=request.input_digest, env=env, output_files=request.output_files, output_directories=request.output_directories, timeout_seconds=request.timeout_seconds, execution_slot_variable=request.execution_slot_variable, ) return await Get( Process, UncacheableProcess(process)) if request.uncacheable else process
async def analyze_first_party_package( request: FirstPartyPkgAnalysisRequest, analyzer: PackageAnalyzerSetup, golang_subsystem: GolangSubsystem, ) -> FallibleFirstPartyPkgAnalysis: wrapped_target, import_path_info, owning_go_mod = await MultiGet( Get(WrappedTarget, Address, request.address), Get(FirstPartyPkgImportPath, FirstPartyPkgImportPathRequest(request.address)), Get(OwningGoMod, OwningGoModRequest(request.address)), ) go_mod_info = await Get(GoModInfo, GoModInfoRequest(owning_go_mod.address)) pkg_sources = await Get( HydratedSources, HydrateSourcesRequest(wrapped_target.target[GoPackageSourcesField]), ) input_digest = await Get(Digest, MergeDigests([pkg_sources.snapshot.digest, analyzer.digest])) result = await Get( FallibleProcessResult, Process( (analyzer.path, request.address.spec_path or "."), input_digest=input_digest, description=f"Determine metadata for {request.address}", level=LogLevel.DEBUG, env={"CGO_ENABLED": "0"}, ), ) return FallibleFirstPartyPkgAnalysis.from_process_result( result, dir_path=request.address.spec_path, import_path=import_path_info.import_path, minimum_go_version=go_mod_info.minimum_go_version or "", description_of_source=f"first-party Go package `{request.address}`", )
async def setup_gofmt(setup_request: SetupRequest, goroot: GoRoot) -> Setup: source_files = await Get( SourceFiles, SourceFilesRequest(field_set.sources for field_set in setup_request.request.field_sets), ) source_files_snapshot = ( source_files.snapshot if setup_request.request.prior_formatter_result is None else setup_request.request.prior_formatter_result) argv = ( os.path.join(goroot.path, "bin/gofmt"), "-l" if setup_request.check_only else "-w", *source_files_snapshot.files, ) process = Process( argv=argv, input_digest=source_files_snapshot.digest, output_files=source_files_snapshot.files, description= f"Run gofmt on {pluralize(len(source_files_snapshot.files), 'file')}.", level=LogLevel.DEBUG, ) return Setup(process=process, original_snapshot=source_files_snapshot)
async def run_buf(request: BufRequest, buf: BufSubsystem) -> LintResults: if buf.skip: return LintResults([], linter_name=request.name) transitive_targets = await Get( TransitiveTargets, TransitiveTargetsRequest( (field_set.address for field_set in request.field_sets)), ) all_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest(tgt[ProtobufSourceField] for tgt in transitive_targets.closure if tgt.has_field(ProtobufSourceField)), ) target_stripped_sources_request = Get( StrippedSourceFiles, SourceFilesRequest( (field_set.sources for field_set in request.field_sets), for_sources_types=(ProtobufSourceField, ), enable_codegen=True, ), ) download_buf_get = Get(DownloadedExternalTool, ExternalToolRequest, buf.get_request(Platform.current)) target_sources_stripped, all_sources_stripped, downloaded_buf = await MultiGet( target_stripped_sources_request, all_stripped_sources_request, download_buf_get) input_digest = await Get( Digest, MergeDigests(( target_sources_stripped.snapshot.digest, all_sources_stripped.snapshot.digest, downloaded_buf.digest, )), ) process_result = await Get( FallibleProcessResult, Process( argv=[ downloaded_buf.exe, "lint", *buf.args, "--path", ",".join(target_sources_stripped.snapshot.files), ], input_digest=input_digest, description= f"Run Buf on {pluralize(len(request.field_sets), 'file')}.", level=LogLevel.DEBUG, ), ) result = LintResult.from_fallible_process_result(process_result) return LintResults([result], linter_name=request.name)
def build_image( self, tags: tuple[str, ...], digest: Digest, dockerfile: str | None = None, build_args: DockerBuildArgs | None = None, env: Mapping[str, str] | None = None, ) -> Process: args = [self.path, "build"] for tag in tags: args.extend(["-t", tag]) if build_args: for build_arg in build_args: args.extend(["--build-arg", build_arg]) if dockerfile: args.extend(["-f", dockerfile]) # Add build context root. args.append(".") return Process( argv=tuple(args), description=(f"Building docker image {tags[0]}" + (f" +{pluralize(len(tags)-1, 'additional tag')}." if len(tags) > 1 else ".")), env=env, input_digest=digest, cache_scope=ProcessCacheScope.PER_SESSION, )
def test_create_from_snapshot_with_env(self): req = Process( argv=("foo", ), description="Some process", env={"VAR": "VAL"}, ) self.assertEqual(req.env, ("VAR", "VAL"))
def test_write_file(self): request = Process( argv=("/bin/bash", "-c", "echo -n 'European Burmese' > roland"), description="echo roland", output_files=("roland", ), ) process_result = self.request_single_product(ProcessResult, request) self.assertEqual( process_result.output_digest, Digest( fingerprint= "63949aa823baf765eff07b946050d76ec0033144c785a94d3ebd82baa931cd16", serialized_bytes_length=80, ), ) files_content_result = self.request_single_product( FilesContent, process_result.output_digest, ) self.assertEqual(files_content_result.dependencies, (FileContent("roland", b"European Burmese", False), ))
def test_pex_execution(rule_runner: RuleRunner, pex_type: type[Pex | VenvPex], internal_only: bool) -> None: sources = rule_runner.request( Digest, [ CreateDigest(( FileContent("main.py", b'print("from main")'), FileContent("subdir/sub.py", b'print("from sub")'), )), ], ) pex_data = create_pex_and_get_all_data( rule_runner, pex_type=pex_type, internal_only=internal_only, main=EntryPoint("main"), sources=sources, ) assert "pex" not in pex_data.files assert "main.py" in pex_data.files assert "subdir/sub.py" in pex_data.files # This should run the Pex using the same interpreter used to create it. We must set the `PATH` # so that the shebang works. pex_exe = (f"./{pex_data.sandbox_path}" if pex_data.is_zipapp else os.path.join(pex_data.sandbox_path, "__main__.py")) process = Process( argv=(pex_exe, ), env={"PATH": os.getenv("PATH", "")}, input_digest=pex_data.pex.digest, description="Run the pex and make sure it works", ) result = rule_runner.request(ProcessResult, [process]) assert result.stdout == b"from main\n"
async def setup_venv_pex_process( request: VenvPexProcess, pex_environment: PexEnvironment ) -> Process: venv_pex = request.venv_pex pex_bin = ( os.path.relpath(venv_pex.pex.argv0, request.working_directory) if request.working_directory else venv_pex.pex.argv0 ) argv = (pex_bin, *request.argv) input_digest = ( await Get(Digest, MergeDigests((venv_pex.digest, request.input_digest))) if request.input_digest else venv_pex.digest ) return Process( argv=argv, description=request.description, level=request.level, input_digest=input_digest, working_directory=request.working_directory, env=request.extra_env, output_files=request.output_files, output_directories=request.output_directories, append_only_caches=pex_environment.in_sandbox( working_directory=request.working_directory ).append_only_caches, timeout_seconds=request.timeout_seconds, execution_slot_variable=request.execution_slot_variable, cache_scope=request.cache_scope, )
def build_image( self, tags: tuple[str, ...], digest: Digest, dockerfile: str, build_args: DockerBuildArgs, context_root: str, env: Mapping[str, str], extra_args: tuple[str, ...] = (), ) -> Process: args = [self.path, "build", *extra_args] for tag in tags: args.extend(["--tag", tag]) for build_arg in build_args: args.extend(["--build-arg", build_arg]) args.extend(["--file", dockerfile]) # Docker context root. args.append(context_root) return Process( argv=tuple(args), description=(f"Building docker image {tags[0]}" + (f" +{pluralize(len(tags)-1, 'additional tag')}." if len(tags) > 1 else "")), env=self._get_process_environment(env), input_digest=digest, immutable_input_digests=self.extra_input_digests, cache_scope=ProcessCacheScope.PER_SESSION, )
async def parse_python_imports(request: ParsePythonImportsRequest) -> ParsedPythonImports: python_interpreter, script_digest, stripped_sources = await MultiGet( Get(PythonExecutable, PexInterpreterConstraints, request.interpreter_constraints), Get(Digest, CreateDigest([FileContent("__parse_python_imports.py", _SCRIPT.encode())])), Get(StrippedSourceFiles, SourceFilesRequest([request.sources])), ) input_digest = await Get( Digest, MergeDigests([script_digest, stripped_sources.snapshot.digest]) ) process_result = await Get( ProcessResult, Process( argv=[ python_interpreter.path, "./__parse_python_imports.py", *stripped_sources.snapshot.files, ], input_digest=input_digest, description=f"Determine Python imports for {request.sources.address}", level=LogLevel.DEBUG, ), ) explicit_imports, _, string_imports = process_result.stdout.decode().partition("--") return ParsedPythonImports( explicit_imports=FrozenOrderedSet(explicit_imports.strip().splitlines()), string_imports=FrozenOrderedSet(string_imports.strip().splitlines()), )
def test_streaming_workunits_expanded_specs(run_tracker: RunTracker) -> None: rule_runner = RuleRunner( target_types=[PythonLibrary], rules=[ QueryRule(ProcessResult, (Process,)), ], ) rule_runner.set_options(["--backend-packages=pants.backend.python"]) rule_runner.create_file("src/python/somefiles/BUILD", "python_library()") rule_runner.create_file("src/python/somefiles/a.py", "print('')") rule_runner.create_file("src/python/somefiles/b.py", "print('')") rule_runner.create_file("src/python/others/BUILD", "python_library()") rule_runner.create_file("src/python/others/a.py", "print('')") rule_runner.create_file("src/python/others/b.py", "print('')") specs = SpecsParser(get_buildroot()).parse_specs( ["src/python/somefiles::", "src/python/others/b.py"] ) class Callback(WorkunitsCallback): @property def can_finish_async(self) -> bool: return False def __call__(self, **kwargs) -> None: context = kwargs["context"] assert isinstance(context, StreamingWorkunitContext) expanded = context.get_expanded_specs() targets = expanded.targets assert len(targets.keys()) == 2 assert targets["src/python/others/b.py"] == [ TargetInfo(filename="src/python/others/b.py") ] assert set(targets["src/python/somefiles"]) == { TargetInfo(filename="src/python/somefiles/a.py"), TargetInfo(filename="src/python/somefiles/b.py"), } handler = StreamingWorkunitHandler( scheduler=rule_runner.scheduler, run_tracker=run_tracker, callbacks=[Callback()], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.INFO, specs=specs, options_bootstrapper=create_options_bootstrapper( ["--backend-packages=pants.backend.python"] ), pantsd=False, ) stdout_process = Process( argv=("/bin/bash", "-c", "/bin/echo 'stdout output'"), description="Stdout process" ) with handler: rule_runner.request(ProcessResult, [stdout_process])
def test_pex_execution(self) -> None: sources_content = InputFilesContent(( FileContent(path="main.py", content=b'print("from main")'), FileContent(path="subdir/sub.py", content=b'print("from sub")'), )) sources = self.request_single_product(Digest, sources_content) pex_output = self.create_pex_and_get_all_data(entry_point="main", sources=sources) pex_files = pex_output["files"] assert "pex" not in pex_files assert "main.py" in pex_files assert "subdir/sub.py" in pex_files init_subsystem(PythonSetup) python_setup = PythonSetup.global_instance() env = { "PATH": create_path_env_var(python_setup.interpreter_search_paths) } process = Process( argv=("python", "test.pex"), env=env, input_files=pex_output["pex"].digest, description="Run the pex and make sure it works", ) result = self.request_single_product(ProcessResult, process) assert result.stdout == b"from main\n"
async def loose_classfiles(classpath_entry: ClasspathEntry, unzip_binary: UnzipBinary) -> LooseClassfiles: dest_dir = "dest" process_results = await MultiGet( Get( ProcessResult, Process( argv=[ unzip_binary.path, "-d", dest_dir, filename, ], output_directories=(dest_dir, ), description=f"Extract {filename}", immutable_input_digests=dict( ClasspathEntry.immutable_inputs([classpath_entry])), level=LogLevel.TRACE, ), ) for filename in ClasspathEntry.immutable_inputs_args([classpath_entry]) ) merged_digest = await Get( Digest, MergeDigests(pr.output_digest for pr in process_results)) return LooseClassfiles(await Get(Digest, RemovePrefix(merged_digest, dest_dir)))
def _deploy_jar_test(rule_runner: RuleRunner, target_name: str) -> None: tgt = rule_runner.get_target(Address("", target_name=target_name)) fat_jar = rule_runner.request( BuiltPackage, [DeployJarFieldSet.create(tgt)], ) jdk_setup = rule_runner.request(JdkSetup, []) bash = rule_runner.request(BashBinary, []) input_digests = rule_runner.request( Digest, [MergeDigests([jdk_setup.digest, fat_jar.digest])]) process_result = rule_runner.request( ProcessResult, [ Process( argv=jdk_setup.args(bash, []) + ("-jar", "dave.jar"), description="Run that test jar", input_digest=input_digests, append_only_caches=jdk_setup.append_only_caches, env=jdk_setup.env, ) ], ) assert process_result.stdout.decode("utf-8").strip() == "Hello, World!"
async def count_loc( console: Console, succinct_code_counter: SuccinctCodeCounter, specs_snapshot: SpecsSnapshot, ) -> CountLinesOfCode: if not specs_snapshot.snapshot.files: return CountLinesOfCode(exit_code=0) scc_program = await Get( DownloadedExternalTool, ExternalToolRequest, succinct_code_counter.get_request(Platform.current), ) input_digest = await Get( Digest, MergeDigests((scc_program.digest, specs_snapshot.snapshot.digest))) result = await Get( ProcessResult, Process( argv=(scc_program.exe, *succinct_code_counter.args), input_digest=input_digest, description= (f"Count lines of code for {pluralize(len(specs_snapshot.snapshot.files), 'file')}" ), level=LogLevel.DEBUG, ), ) console.print_stdout(result.stdout.decode()) return CountLinesOfCode(exit_code=0)
async def generate_testmain( request: GenerateTestMainRequest, analyzer: AnalyzerSetup ) -> GeneratedTestMain: input_digest = await Get(Digest, MergeDigests([request.digest, analyzer.digest])) test_paths = tuple(f"{GeneratedTestMain.TEST_PKG}:{path}" for path in request.test_paths) xtest_paths = tuple(f"{GeneratedTestMain.XTEST_PKG}:{path}" for path in request.xtest_paths) result = await Get( ProcessResult, Process( argv=(analyzer.PATH, request.import_path, *test_paths, *xtest_paths), input_digest=input_digest, description=f"Analyze Go test sources for {request.import_path}", level=LogLevel.DEBUG, output_files=("_testmain.go",), ), ) metadata = json.loads(result.stdout.decode("utf-8")) return GeneratedTestMain( digest=result.output_digest, has_tests=metadata["has_tests"], has_xtests=metadata["has_xtests"], )
async def create_archive(request: CreateArchive) -> Digest: if request.format == ArchiveFormat.ZIP: zip_binary = await Get(ZipBinary, _ZipBinaryRequest()) argv = zip_binary.create_archive_argv(request) env = {} input_digest = request.snapshot.digest else: tar_binary = await Get(TarBinary, _TarBinaryRequest()) argv = tar_binary.create_archive_argv(request) # `tar` expects to find a couple binaries like `gzip` and `xz` by looking on the PATH. env = {"PATH": os.pathsep.join(SEARCH_PATHS)} # `tar` requires that the output filename's parent directory exists. output_dir_digest = await Get( Digest, CreateDigest([Directory(os.path.dirname(request.output_filename)) ])) input_digest = await Get( Digest, MergeDigests([output_dir_digest, request.snapshot.digest])) result = await Get( ProcessResult, Process( argv=argv, env=env, input_digest=input_digest, description=f"Create {request.output_filename}", level=LogLevel.DEBUG, output_files=(request.output_filename, ), ), ) return result.output_digest
async def setup_pex_cli_process( request: PexCliProcess, pex_binary: PexBinary, pex_environment: PexEnvironment, python_native_code: PythonNativeCode, ) -> Process: downloaded_pex_bin = await Get(DownloadedExternalTool, ExternalToolRequest, pex_binary.get_request(Platform.current)) input_digest = (await Get( Digest, MergeDigests([ request.additional_input_digest, downloaded_pex_bin.digest ])) if request.additional_input_digest else downloaded_pex_bin.digest) pex_root_path = ".cache/pex_root" argv = pex_environment.create_argv(downloaded_pex_bin.exe, *request.argv, "--pex-root", pex_root_path) env = { **pex_environment.environment_dict, **python_native_code.environment_dict, **(request.extra_env or {}), } return Process( argv, description=request.description, input_digest=input_digest, env=env, output_files=request.output_files, output_directories=request.output_directories, append_only_caches={"pex_root": pex_root_path}, )