def create_pex(request: CreatePex, pex_bin: DownloadedPexBin, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, pex_build_environment: PexBuildEnvironment, platform: Platform) -> Pex: """Returns a PEX with the given requirements, optional entry point, and optional interpreter constraints.""" interpreter_constraint_args = [] for constraint in request.interpreter_constraints: interpreter_constraint_args.extend( ["--interpreter-constraint", constraint]) argv = ["--output-file", request.output_filename] if request.entry_point is not None: argv.extend(["--entry-point", request.entry_point]) argv.extend(interpreter_constraint_args + list(request.requirements)) source_dir_name = 'source_files' argv.append(f'--sources-directory={source_dir_name}') sources_digest = request.input_files_digest if request.input_files_digest else EMPTY_DIRECTORY_DIGEST sources_digest_as_subdir = yield Get( Digest, DirectoryWithPrefixToAdd(sources_digest, source_dir_name)) all_inputs = ( pex_bin.directory_digest, sources_digest_as_subdir, ) merged_digest = yield Get(Digest, DirectoriesToMerge(directories=all_inputs)) # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without # cross-building we specify that out PEX command be run on the current local platform. When we # support cross-building through CLI flags we can configure requests that build a PEX for out # local platform that are able to execute on a different platform, but for now in order to # guarantee correct build we need to restrict this command to execute on the same platform type # that the output is intended for. The correct way to interpret the keys # (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of # this command is intended for `target_platform_constraint` iff it is run on `execution_platform # constraint`". execute_process_request = MultiPlatformExecuteProcessRequest( { (PlatformConstraint(platform.value), PlatformConstraint(platform.value)): pex_bin.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_build_environment=pex_build_environment, pex_args=argv, input_files=merged_digest, description= f"Create a requirements PEX: {', '.join(request.requirements)}", output_files=(request.output_filename, )) }) result = yield Get(ExecuteProcessResult, MultiPlatformExecuteProcessRequest, execute_process_request) yield Pex(directory_digest=result.output_directory_digest)
async def merge_coverage_data( data_batch: PytestCoverageDataBatch, transitive_targets: TransitiveHydratedTargets, python_setup: PythonSetup, coverage_setup: CoverageSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> MergedCoverageData: """Takes all python test results and merges their coverage data into a single sql file.""" # We start with a bunch of test results, each of which has a coverage data file called `.coverage` # We prefix each of these with their address so that we can write them all into a single pex. coverage_directory_digests = await MultiGet( Get[Digest]( DirectoryWithPrefixToAdd( directory_digest=result.test_result.coverage_data. digest, # type: ignore[attr-defined] prefix=result.address.path_safe_spec, )) for result in data_batch.addresses_and_test_results if result.test_result is not None and result.test_result.coverage_data is not None) sources = await Get[SourceFiles](AllSourceFilesRequest( (ht.adaptor for ht in transitive_targets.closure), strip_source_roots=False)) sources_with_inits_snapshot = await Get[InitInjectedSnapshot]( InjectInitRequest(sources.snapshot)) coveragerc = await Get[Coveragerc](CoveragercRequest(HydratedTargets( transitive_targets.closure), test_time=True)) merged_input_files: Digest = await Get( Digest, DirectoriesToMerge(directories=( *coverage_directory_digests, sources_with_inits_snapshot.snapshot.directory_digest, coveragerc.digest, coverage_setup.requirements_pex.directory_digest, )), ) prefixes = [ f"{result.address.path_safe_spec}/.coverage" for result in data_batch.addresses_and_test_results ] coverage_args = ["combine", *prefixes] request = coverage_setup.requirements_pex.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./{coverage_setup.requirements_pex.output_filename}", pex_args=coverage_args, input_files=merged_input_files, output_files=(".coverage", ), description=f"Merge coverage reports.", ) result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request) return MergedCoverageData(coverage_data=result.output_directory_digest)
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot: if request.py2: # TODO: Implement Python 2 support. This will involve, among other things: merging ancestor # __init__.py files into the chroot, detecting packages based on the presence of __init__.py, # and inspecting all __init__.py files for the namespace package incantation. raise UnsupportedPythonVersion('Running setup.py commands not supported for Python 2.') owned_deps = await Get[OwnedDependencies](DependencyOwner(request.exported_target)) targets = HydratedTargets(od.hydrated_target for od in owned_deps) sources = await Get[SetupPySources](SetupPySourcesRequest(targets)) requirements = await Get[ExportedTargetRequirements](DependencyOwner(request.exported_target)) # Nest the sources under the src/ prefix. src_digest = await Get[Digest](DirectoryWithPrefixToAdd(sources.digest, CHROOT_SOURCE_ROOT)) # Generate the kwargs to the setup() call. setup_kwargs = request.exported_target.hydrated_target.adaptor.provides.setup_py_keywords.copy() setup_kwargs.update({ 'package_dir': {'': CHROOT_SOURCE_ROOT}, 'packages': sources.packages, 'namespace_packages': sources.namespace_packages, 'package_data': dict(sources.package_data), 'install_requires': requirements.requirement_strs }) ht = request.exported_target.hydrated_target key_to_binary_spec = getattr(ht.adaptor.provides, 'binaries', {}) keys = list(key_to_binary_spec.keys()) binaries = await MultiGet(Get[HydratedTarget]( Address, Address.parse(key_to_binary_spec[key], relative_to=ht.address.spec_path)) for key in keys) for key, binary in zip(keys, binaries): if (not isinstance(binary.adaptor, PythonBinaryAdaptor) or getattr(binary.adaptor, 'entry_point', None) is None): raise InvalidEntryPoint( f'The binary {key} exported by {ht.address.reference()} is not a valid entry point.') entry_points = setup_kwargs['entry_points'] = setup_kwargs.get('entry_points', {}) console_scripts = entry_points['console_scripts'] = entry_points.get('console_scripts', []) console_scripts.append(f'{key}={binary.adaptor.entry_point}') # Generate the setup script. setup_py_content = SETUP_BOILERPLATE.format( target_address_spec=ht.address.reference(), setup_kwargs_str=distutils_repr(setup_kwargs) ).encode() extra_files_digest = await Get[Digest]( InputFilesContent([ FileContent('setup.py', setup_py_content), FileContent('MANIFEST.in', 'include *.py'.encode()) # Make sure setup.py is included. ])) chroot_digest = await Get[Digest](DirectoriesToMerge((src_digest, extra_files_digest))) return SetupPyChroot(chroot_digest, json.dumps(setup_kwargs, sort_keys=True))
async def merge_coverage_data( data_collection: PytestCoverageDataCollection, coverage_setup: CoverageSetup, transitive_targets: TransitiveTargets, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> MergedCoverageData: """Takes all Python test results and merges their coverage data into a single SQL file.""" # We start with a bunch of test results, each of which has a coverage data file called # `.coverage`. We prefix each of these with their address so that we can write them all into a # single PEX. coverage_directory_digests = await MultiGet( Get[Digest](DirectoryWithPrefixToAdd(data.digest, prefix=data.address.path_safe_spec)) for data in data_collection ) sources = await Get[SourceFiles]( AllSourceFilesRequest( (tgt.get(Sources) for tgt in transitive_targets.closure), strip_source_roots=False ) ) sources_with_inits = await Get[InitInjectedSnapshot](InjectInitRequest(sources.snapshot)) coverage_config = await Get[CoverageConfig]( CoverageConfigRequest(Targets(transitive_targets.closure), is_test_time=True) ) merged_input_files: Digest = await Get( Digest, DirectoriesToMerge( directories=( *coverage_directory_digests, sources_with_inits.snapshot.directory_digest, coverage_config.digest, coverage_setup.requirements_pex.directory_digest, ) ), ) prefixes = [f"{data.address.path_safe_spec}/.coverage" for data in data_collection] coverage_args = ("combine", *prefixes) process = coverage_setup.requirements_pex.create_process( pex_path=f"./{coverage_setup.requirements_pex.output_filename}", pex_args=coverage_args, input_files=merged_input_files, output_files=(".coverage",), description=f"Merge {len(prefixes)} Pytest coverage reports.", python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, ) result = await Get[ProcessResult](Process, process) return MergedCoverageData(coverage_data=result.output_directory_digest)
def test_add_prefix(self): input_files_content = InputFilesContent(( FileContent(path='main.py', content=b'print("from main")', is_executable=False), FileContent(path='subdir/sub.py', content=b'print("from sub")', is_executable=False), )) digest, = self.scheduler.product_request(Digest, [input_files_content]) dpa = DirectoryWithPrefixToAdd(digest, "outer_dir") output_digest, = self.scheduler.product_request(Digest, [dpa]) snapshot, = self.scheduler.product_request(Snapshot, [output_digest]) self.assertEqual(sorted(snapshot.files), ['outer_dir/main.py', 'outer_dir/subdir/sub.py']) self.assertEqual(sorted(snapshot.dirs), ['outer_dir', 'outer_dir/subdir'])
def test_add_prefix(self): input_files_content = InputFilesContent(( FileContent(path='main.py', content=b'print("from main")'), FileContent(path='subdir/sub.py', content=b'print("from sub")'), )) digest = self.request_single_product(Digest, input_files_content) dpa = DirectoryWithPrefixToAdd(digest, "outer_dir") output_digest = self.request_single_product(Digest, dpa) snapshot = self.request_single_product(Snapshot, output_digest) self.assertEqual(sorted(snapshot.files), ['outer_dir/main.py', 'outer_dir/subdir/sub.py']) self.assertEqual(sorted(snapshot.dirs), ['outer_dir', 'outer_dir/subdir'])
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot: exported_target = request.exported_target owned_deps = await Get[OwnedDependencies](DependencyOwner(exported_target)) targets = Targets(od.target for od in owned_deps) sources = await Get[SetupPySources](SetupPySourcesRequest(targets, py2=request.py2)) requirements = await Get[ExportedTargetRequirements]( DependencyOwner(exported_target)) # Nest the sources under the src/ prefix. src_digest = await Get[Digest](DirectoryWithPrefixToAdd( sources.digest, CHROOT_SOURCE_ROOT)) target = exported_target.target provides = exported_target.provides # Generate the kwargs to the setup() call. setup_kwargs = provides.setup_py_keywords.copy() setup_kwargs.update({ "package_dir": { "": CHROOT_SOURCE_ROOT }, "packages": sources.packages, "namespace_packages": sources.namespace_packages, "package_data": dict(sources.package_data), "install_requires": tuple(requirements), }) key_to_binary_spec = provides.binaries keys = list(key_to_binary_spec.keys()) binaries = await Get[Targets](Addresses( Address.parse(key_to_binary_spec[key], relative_to=target.address.spec_path) for key in keys)) for key, binary in zip(keys, binaries): binary_entry_point = binary.get(PythonEntryPoint).value if not binary_entry_point: raise InvalidEntryPoint( f"The binary {key} exported by {target.address.reference()} is not a valid entry " f"point.") entry_points = setup_kwargs["entry_points"] = setup_kwargs.get( "entry_points", {}) console_scripts = entry_points["console_scripts"] = entry_points.get( "console_scripts", []) console_scripts.append(f"{key}={binary_entry_point}") # Generate the setup script. setup_py_content = SETUP_BOILERPLATE.format( target_address_spec=target.address.reference(), setup_kwargs_str=distutils_repr(setup_kwargs), ).encode() extra_files_digest = await Get[Digest]( InputFilesContent([ FileContent("setup.py", setup_py_content), FileContent( "MANIFEST.in", "include *.py".encode()), # Make sure setup.py is included. ])) chroot_digest = await Get[Digest](DirectoriesToMerge( (src_digest, extra_files_digest))) return SetupPyChroot(chroot_digest, json.dumps(setup_kwargs, sort_keys=True))
async def create_pex( request: CreatePex, pex_bin: DownloadedPexBin, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, pex_build_environment: PexBuildEnvironment, platform: Platform, ) -> Pex: """Returns a PEX with the given requirements, optional entry point, optional interpreter constraints, and optional requirement constraints.""" argv = [ "--output-file", request.output_filename, *request.interpreter_constraints.generate_pex_arg_list(), *request.additional_args, ] if python_setup.resolver_jobs: argv.extend(["--jobs", python_setup.resolver_jobs]) if python_setup.manylinux: argv.extend(["--manylinux", python_setup.manylinux]) else: argv.append("--no-manylinux") if request.entry_point is not None: argv.extend(["--entry-point", request.entry_point]) if python_setup.requirement_constraints is not None: argv.extend(["--constraints", python_setup.requirement_constraints]) source_dir_name = "source_files" argv.append(f"--sources-directory={source_dir_name}") argv.extend(request.requirements.requirements) constraint_file_snapshot = EMPTY_SNAPSHOT if python_setup.requirement_constraints is not None: constraint_file_snapshot = await Get[Snapshot](PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin= "the option `--python-setup-requirement-constraints`", )) sources_digest = (request.input_files_digest if request.input_files_digest else EMPTY_DIRECTORY_DIGEST) sources_digest_as_subdir = await Get[Digest](DirectoryWithPrefixToAdd( sources_digest, source_dir_name)) merged_digest = await Get[Digest](DirectoriesToMerge(directories=( pex_bin.directory_digest, sources_digest_as_subdir, constraint_file_snapshot.directory_digest, ))) # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without # cross-building, we specify that our PEX command be run on the current local platform. When we # support cross-building through CLI flags we can configure requests that build a PEX for our # local platform that are able to execute on a different platform, but for now in order to # guarantee correct build we need to restrict this command to execute on the same platform type # that the output is intended for. The correct way to interpret the keys # (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of # this command is intended for `target_platform_constraint` iff it is run on `execution_platform # constraint`". execute_process_request = MultiPlatformExecuteProcessRequest({ ( PlatformConstraint(platform.value), PlatformConstraint(platform.value), ): pex_bin.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_build_environment=pex_build_environment, pex_args=argv, input_files=merged_digest, description= f"Create a requirements PEX: {', '.join(request.requirements.requirements)}", output_files=(request.output_filename, ), ) }) result = await Get[ExecuteProcessResult]( MultiPlatformExecuteProcessRequest, execute_process_request) return Pex(directory_digest=result.output_directory_digest, output_filename=request.output_filename)
async def create_pex( request: PexRequest, pex_bin: DownloadedPexBin, python_setup: PythonSetup, python_repos: PythonRepos, subprocess_encoding_environment: SubprocessEncodingEnvironment, pex_build_environment: PexBuildEnvironment, platform: Platform, log_level: LogLevel, ) -> Pex: """Returns a PEX with the given requirements, optional entry point, optional interpreter constraints, and optional requirement constraints.""" argv = [ "--output-file", request.output_filename, *request.interpreter_constraints.generate_pex_arg_list(), # NB: In setting `--no-pypi`, we rely on the default value of `--python-repos-indexes` # including PyPI, which will override `--no-pypi` and result in using PyPI in the default # case. Why set `--no-pypi`, then? We need to do this so that # `--python-repos-repos=['custom_url']` will only point to that index and not include PyPI. "--no-pypi", *(f"--index={index}" for index in python_repos.indexes), *(f"--repo={repo}" for repo in python_repos.repos), *request.additional_args, ] pex_debug = PexDebug(log_level) argv.extend(pex_debug.iter_pex_args()) if python_setup.resolver_jobs: argv.extend(["--jobs", python_setup.resolver_jobs]) if python_setup.manylinux: argv.extend(["--manylinux", python_setup.manylinux]) else: argv.append("--no-manylinux") if request.entry_point is not None: argv.extend(["--entry-point", request.entry_point]) if python_setup.requirement_constraints is not None: argv.extend(["--constraints", python_setup.requirement_constraints]) source_dir_name = "source_files" argv.append(f"--sources-directory={source_dir_name}") argv.extend(request.requirements.requirements) constraint_file_snapshot = EMPTY_SNAPSHOT if python_setup.requirement_constraints is not None: constraint_file_snapshot = await Get[Snapshot](PathGlobs( [python_setup.requirement_constraints], glob_match_error_behavior=GlobMatchErrorBehavior.error, conjunction=GlobExpansionConjunction.all_match, description_of_origin= "the option `--python-setup-requirement-constraints`", )) sources_digest_as_subdir = await Get[Digest](DirectoryWithPrefixToAdd( request.sources or EMPTY_DIRECTORY_DIGEST, source_dir_name)) additional_inputs_digest = request.additional_inputs or EMPTY_DIRECTORY_DIGEST merged_digest = await Get[Digest](DirectoriesToMerge(directories=( pex_bin.directory_digest, sources_digest_as_subdir, additional_inputs_digest, constraint_file_snapshot.directory_digest, ))) # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without # cross-building, we specify that our PEX command be run on the current local platform. When we # support cross-building through CLI flags we can configure requests that build a PEX for our # local platform that are able to execute on a different platform, but for now in order to # guarantee correct build we need to restrict this command to execute on the same platform type # that the output is intended for. The correct way to interpret the keys # (execution_platform_constraint, target_platform_constraint) of this dictionary is "The output of # this command is intended for `target_platform_constraint` iff it is run on `execution_platform # constraint`". description = request.description if description is None: if request.requirements.requirements: description = f"Resolving {', '.join(request.requirements.requirements)}" else: description = f"Building PEX" execute_process_request = MultiPlatformExecuteProcessRequest({ ( PlatformConstraint(platform.value), PlatformConstraint(platform.value), ): pex_bin.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_build_environment=pex_build_environment, pex_args=argv, input_files=merged_digest, description=description, output_files=(request.output_filename, ), ) }) result = await Get[ExecuteProcessResult]( MultiPlatformExecuteProcessRequest, execute_process_request) if pex_debug.might_log: lines = result.stderr.decode().splitlines() if lines: pex_debug.log( f"Debug output from Pex for: {execute_process_request}") for line in lines: pex_debug.log(line) return Pex(directory_digest=result.output_directory_digest, output_filename=request.output_filename)