async def run_setup_py( req: RunSetupPyRequest, setuptools_setup: SetuptoolsSetup, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment ) -> RunSetupPyResult: """Run a setup.py command on a single exported target.""" merged_input_files = await Get[Digest]( DirectoriesToMerge(directories=( req.chroot.digest, setuptools_setup.requirements_pex.directory_digest)) ) # The setuptools dist dir, created by it under the chroot (not to be confused with # pants's own dist dir, at the buildroot). # TODO: The user can change this with the --dist-dir flag to the sdist and bdist_wheel commands. # See https://github.com/pantsbuild/pants/issues/8912. dist_dir = 'dist/' request = setuptools_setup.requirements_pex.create_execute_request( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./setuptools.pex", pex_args=('setup.py', *req.args), input_files=merged_input_files, # setuptools commands that create dists write them to the distdir. # TODO: Could there be other useful files to capture? output_directories=(dist_dir,), description=f'Run setuptools for {req.exported_target.hydrated_target.address.reference()}', ) result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request) output_digest = await Get[Digest]( DirectoryWithPrefixToStrip(result.output_directory_digest, dist_dir)) return RunSetupPyResult(output_digest)
async def run_setup_py( req: RunSetupPyRequest, setuptools_setup: SetuptoolsSetup, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> RunSetupPyResult: """Run a setup.py command on a single exported target.""" merged_input_files = await Get[Digest](DirectoriesToMerge( directories=(req.chroot.digest, setuptools_setup.requirements_pex.directory_digest))) # The setuptools dist dir, created by it under the chroot (not to be confused with # pants's own dist dir, at the buildroot). dist_dir = "dist/" process = setuptools_setup.requirements_pex.create_process( python_setup=python_setup, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./setuptools.pex", pex_args=("setup.py", *req.args), input_files=merged_input_files, # setuptools commands that create dists write them to the distdir. # TODO: Could there be other useful files to capture? output_directories=(dist_dir, ), description= f"Run setuptools for {req.exported_target.target.address.reference()}", ) result = await Get[ProcessResult](Process, process) output_digest = await Get[Digest](DirectoryWithPrefixToStrip( result.output_directory_digest, dist_dir)) return RunSetupPyResult(output_digest)
async def strip_source_root( hydrated_target: HydratedTarget, source_root_config: SourceRootConfig) -> SourceRootStrippedSources: """Relativize targets to their source root, e.g. `src/python/pants/util/strutil.py` -> `pants/util/strutil.py .""" target_adaptor = hydrated_target.adaptor source_roots = source_root_config.get_source_roots() # TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to # simplify the hasattr() checks here! if not hasattr(target_adaptor, 'sources'): return SourceRootStrippedSources(snapshot=EMPTY_SNAPSHOT) digest = target_adaptor.sources.snapshot.directory_digest source_root = source_roots.find_by_path(target_adaptor.address.spec_path) if source_root is None: # If we found no source root, use the target's dir. # Note that when --source-unmatched is 'create' (the default) we'll never return None, # but will return the target's dir. This check allows this code to work even if # --source-unmatched is 'fail'. source_root_path = target_adaptor.address.spec_path else: source_root_path = source_root.path # Loose `Files`, as opposed to `Resources` or `Target`s, have no (implied) package # structure and so we do not remove their source root like we normally do, so that filesystem # APIs may still access the files. See pex_build_util.py's `_create_source_dumper`. if target_adaptor.type_alias == Files.alias(): source_root_path = '' resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip( directory_digest=digest, prefix=source_root_path)) resulting_snapshot = await Get[Snapshot](Digest, resulting_digest) return SourceRootStrippedSources(snapshot=resulting_snapshot)
async def strip_source_root( hydrated_target: HydratedTarget, source_root_config: SourceRootConfig) -> SourceRootStrippedSources: """Relativize targets to their source root, e.g. `src/python/pants/util/strutil.py` -> `pants/util/strutil.py .""" target_adaptor = hydrated_target.adaptor source_roots = source_root_config.get_source_roots() # TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to # simplify the hasattr() checks here! if not hasattr(target_adaptor, 'sources'): return SourceRootStrippedSources(snapshot=EMPTY_SNAPSHOT) digest = target_adaptor.sources.snapshot.directory_digest source_root = source_roots.find_by_path(target_adaptor.address.spec_path) # Loose `Files`, as opposed to `Resources` or `Target`s, have no (implied) package # structure and so we do not remove their source root like we normally do, so that filesystem # APIs may still access the files. See pex_build_util.py's `_create_source_dumper`. if target_adaptor.type_alias == Files.alias(): source_root = None resulting_digest = await Get( Digest, DirectoryWithPrefixToStrip( directory_digest=digest, prefix=source_root.path if source_root else "")) resulting_snapshot = await Get(Snapshot, Digest, resulting_digest) return SourceRootStrippedSources(snapshot=resulting_snapshot)
async def strip_source_roots_from_snapshot( request: StripSnapshotRequest, source_root_config: SourceRootConfig, ) -> SourceRootStrippedSources: """Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` -> `pants/util/strutil.py`.""" source_roots_object = source_root_config.get_source_roots() def determine_source_root(path: str) -> str: source_root = source_roots_object.safe_find_by_path(path) if source_root is not None: return cast(str, source_root.path) if source_root_config.options.unmatched == "fail": raise NoSourceRootError( f"Could not find a source root for `{path}`.") # Otherwise, create a source root by using the parent directory. return PurePath(path).parent.as_posix() if request.representative_path is not None: resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip( directory_digest=request.snapshot.directory_digest, prefix=determine_source_root(request.representative_path), )) resulting_snapshot = await Get[Snapshot](Digest, resulting_digest) return SourceRootStrippedSources(snapshot=resulting_snapshot) files_grouped_by_source_root = { source_root: tuple(files) for source_root, files in itertools.groupby(request.snapshot.files, key=determine_source_root) } snapshot_subsets = await MultiGet(Get[Snapshot](SnapshotSubset( directory_digest=request.snapshot.directory_digest, globs=PathGlobs(files), )) for files in files_grouped_by_source_root.values()) resulting_digests = await MultiGet( Get[Digest](DirectoryWithPrefixToStrip( directory_digest=snapshot.directory_digest, prefix=source_root)) for snapshot, source_root in zip(snapshot_subsets, files_grouped_by_source_root.keys())) merged_result = await Get[Digest](DirectoriesToMerge(resulting_digests)) resulting_snapshot = await Get[Snapshot](Digest, merged_result) return SourceRootStrippedSources(resulting_snapshot)
async def get_ancestor_init_py( targets: Targets, source_root_config: SourceRootConfig ) -> AncestorInitPyFiles: """Find any ancestor __init__.py files for the given targets. Includes sibling __init__.py files. Returns the files stripped of their source roots. """ source_roots = source_root_config.get_source_roots() sources = await Get[SourceFiles]( AllSourceFilesRequest(tgt[PythonSources] for tgt in targets if tgt.has_field(PythonSources)) ) # Find the ancestors of all dirs containing .py files, including those dirs themselves. source_dir_ancestors: Set[Tuple[str, str]] = set() # Items are (src_root, path incl. src_root). for fp in sources.snapshot.files: source_dir_ancestor = os.path.dirname(fp) source_root = source_root_or_raise(source_roots, fp) # Do not allow the repository root to leak (i.e., '.' should not be a package in setup.py). while source_dir_ancestor != source_root: source_dir_ancestors.add((source_root, source_dir_ancestor)) source_dir_ancestor = os.path.dirname(source_dir_ancestor) source_dir_ancestors_list = list(source_dir_ancestors) # To force a consistent order. # Note that we must MultiGet single globs instead of a a single Get for all the globs, because # we match each result to its originating glob (see use of zip below). ancestor_init_py_snapshots = await MultiGet[Snapshot]( Get[Snapshot](PathGlobs, PathGlobs([os.path.join(source_dir_ancestor[1], "__init__.py")])) for source_dir_ancestor in source_dir_ancestors_list ) source_root_stripped_ancestor_init_pys = await MultiGet[Digest]( Get[Digest]( DirectoryWithPrefixToStrip( directory_digest=snapshot.directory_digest, prefix=source_dir_ancestor[0] ) ) for snapshot, source_dir_ancestor in zip( ancestor_init_py_snapshots, source_dir_ancestors_list ) ) return AncestorInitPyFiles(source_root_stripped_ancestor_init_pys)
def test_strip_prefix(self): # Set up files: relevant_files = ( 'characters/dark_tower/roland', 'characters/dark_tower/susannah', ) all_files = ( 'books/dark_tower/gunslinger', 'characters/altered_carbon/kovacs', ) + relevant_files + ( 'index', ) with temporary_dir() as temp_dir: safe_file_dump(os.path.join(temp_dir, 'index'), 'books\ncharacters\n') safe_file_dump( os.path.join(temp_dir, "characters", "altered_carbon", "kovacs"), "Envoy", makedirs=True, ) tower_dir = os.path.join(temp_dir, "characters", "dark_tower") safe_file_dump(os.path.join(tower_dir, "roland"), "European Burmese", makedirs=True) safe_file_dump(os.path.join(tower_dir, "susannah"), "Not sure actually", makedirs=True) safe_file_dump( os.path.join(temp_dir, "books", "dark_tower", "gunslinger"), "1982", makedirs=True, ) snapshot, snapshot_with_extra_files = self.scheduler.capture_snapshots(( PathGlobsAndRoot(PathGlobs(("characters/dark_tower/*",)), temp_dir), PathGlobsAndRoot(PathGlobs(("**",)), temp_dir), )) # Check that we got the full snapshots that we expect self.assertEquals(snapshot.files, relevant_files) self.assertEquals(snapshot_with_extra_files.files, all_files) # Strip empty prefix: zero_prefix_stripped_digest = assert_single_element(self.scheduler.product_request( Digest, [DirectoryWithPrefixToStrip(snapshot.directory_digest, "")], )) self.assertEquals(snapshot.directory_digest, zero_prefix_stripped_digest) # Strip a non-empty prefix shared by all files: stripped_digest = assert_single_element(self.scheduler.product_request( Digest, [DirectoryWithPrefixToStrip(snapshot.directory_digest, "characters/dark_tower")], )) self.assertEquals( stripped_digest, Digest( fingerprint='71e788fc25783c424db555477071f5e476d942fc958a5d06ffc1ed223f779a8c', serialized_bytes_length=162, ) ) expected_snapshot = assert_single_element(self.scheduler.capture_snapshots(( PathGlobsAndRoot(PathGlobs(("*",)), tower_dir), ))) self.assertEquals(expected_snapshot.files, ('roland', 'susannah')) self.assertEquals(stripped_digest, expected_snapshot.directory_digest) # Try to strip a prefix which isn't shared by all files: with self.assertRaisesWithMessageContaining(Exception, "Cannot strip prefix characters/dark_tower from root directory Digest(Fingerprint<28c47f77867f0c8d577d2ada2f06b03fc8e5ef2d780e8942713b26c5e3f434b8>, 243) - root directory contained non-matching directory named: books and file named: index"): self.scheduler.product_request( Digest, [DirectoryWithPrefixToStrip(snapshot_with_extra_files.directory_digest, "characters/dark_tower")] )
def run_python_test(test_target, pytest, python_setup, source_root_config, subprocess_encoding_environment): """Runs pytest for one target.""" # TODO(7726): replace this with a proper API to get the `closure` for a # TransitiveHydratedTarget. transitive_hydrated_targets = yield Get( TransitiveHydratedTargets, BuildFileAddresses((test_target.address, ))) all_targets = [t.adaptor for t in transitive_hydrated_targets.closure] interpreter_constraints = { constraint for target_adaptor in all_targets for constraint in python_setup.compatibility_or_constraints( getattr(target_adaptor, 'compatibility', None)) } # Produce a pex containing pytest and all transitive 3rdparty requirements. output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex' all_target_requirements = [] for maybe_python_req_lib in all_targets: # This is a python_requirement()-like target. if hasattr(maybe_python_req_lib, 'requirement'): all_target_requirements.append( str(maybe_python_req_lib.requirement)) # This is a python_requirement_library()-like target. if hasattr(maybe_python_req_lib, 'requirements'): for py_req in maybe_python_req_lib.requirements: all_target_requirements.append(str(py_req.requirement)) all_requirements = all_target_requirements + list( pytest.get_requirement_strings()) resolved_requirements_pex = yield Get( ResolvedRequirementsPex, ResolveRequirementsRequest( output_filename=output_pytest_requirements_pex_filename, requirements=tuple(sorted(all_requirements)), interpreter_constraints=tuple(sorted(interpreter_constraints)), entry_point="pytest:main", )) source_roots = source_root_config.get_source_roots() # Gather sources and adjust for the source root. # TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to # simplify the hasattr() checks here! # TODO(7714): restore the full source name for the stdout of the Pytest run. sources_snapshots_and_source_roots = [] for maybe_source_target in all_targets: if hasattr(maybe_source_target, 'sources'): tgt_snapshot = maybe_source_target.sources.snapshot tgt_source_root = source_roots.find_by_path( maybe_source_target.address.spec_path) sources_snapshots_and_source_roots.append( (tgt_snapshot, tgt_source_root)) all_sources_digests = yield [ Get( Digest, DirectoryWithPrefixToStrip( directory_digest=snapshot.directory_digest, prefix=source_root.path)) for snapshot, source_root in sources_snapshots_and_source_roots ] sources_digest = yield Get( Digest, DirectoriesToMerge(directories=tuple(all_sources_digests)), ) inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest) all_input_digests = [ sources_digest, inits_digest.directory_digest, resolved_requirements_pex.directory_digest, ] merged_input_files = yield Get( Digest, DirectoriesToMerge, DirectoriesToMerge(directories=tuple(all_input_digests)), ) interpreter_search_paths = create_path_env_var( python_setup.interpreter_search_paths) pex_exe_env = { 'PATH': interpreter_search_paths, **subprocess_encoding_environment.invocation_environment_dict } # NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like # `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote # execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable # somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not # necessarily the interpreter that PEX will use to execute the generated .pex file. # TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target # platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735. request = ExecuteProcessRequest( argv=("python", './{}'.format(output_pytest_requirements_pex_filename)), env=pex_exe_env, input_files=merged_input_files, description='Run pytest for {}'.format( test_target.address.reference()), ) result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, request) status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE yield TestResult( status=status, stdout=result.stdout.decode(), stderr=result.stderr.decode(), )
def run_python_test(test_target, pytest, python_setup, source_root_config, subprocess_encoding_environment): """Runs pytest for one target.""" # TODO(7726): replace this with a proper API to get the `closure` for a # TransitiveHydratedTarget. transitive_hydrated_targets = yield Get( TransitiveHydratedTargets, BuildFileAddresses((test_target.address,)) ) all_targets = [t.adaptor for t in transitive_hydrated_targets.closure] interpreter_constraints = { constraint for target_adaptor in all_targets for constraint in python_setup.compatibility_or_constraints( getattr(target_adaptor, 'compatibility', None) ) } # Produce a pex containing pytest and all transitive 3rdparty requirements. output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex' all_target_requirements = [] for maybe_python_req_lib in all_targets: # This is a python_requirement()-like target. if hasattr(maybe_python_req_lib, 'requirement'): all_target_requirements.append(str(maybe_python_req_lib.requirement)) # This is a python_requirement_library()-like target. if hasattr(maybe_python_req_lib, 'requirements'): for py_req in maybe_python_req_lib.requirements: all_target_requirements.append(str(py_req.requirement)) all_requirements = all_target_requirements + list(pytest.get_requirement_strings()) resolved_requirements_pex = yield Get( RequirementsPex, RequirementsPexRequest( output_filename=output_pytest_requirements_pex_filename, requirements=tuple(sorted(all_requirements)), interpreter_constraints=tuple(sorted(interpreter_constraints)), entry_point="pytest:main", ) ) # Gather sources and adjust for source roots. # TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to # simplify the hasattr() checks here! source_roots = source_root_config.get_source_roots() sources_digest_to_source_roots: Dict[Digest, Optional[SourceRoot]] = {} for maybe_source_target in all_targets: if not hasattr(maybe_source_target, 'sources'): continue digest = maybe_source_target.sources.snapshot.directory_digest source_root = source_roots.find_by_path(maybe_source_target.address.spec_path) if maybe_source_target.type_alias == Files.alias(): # Loose `Files`, as opposed to `Resources` or `PythonTarget`s, have no (implied) package # structure and so we do not remove their source root like we normally do, so that Python # filesystem APIs may still access the files. See pex_build_util.py's `_create_source_dumper`. source_root = None sources_digest_to_source_roots[digest] = source_root.path if source_root else "" stripped_sources_digests = yield [ Get(Digest, DirectoryWithPrefixToStrip(directory_digest=digest, prefix=source_root)) for digest, source_root in sources_digest_to_source_roots.items() ] sources_digest = yield Get( Digest, DirectoriesToMerge(directories=tuple(stripped_sources_digests)), ) inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest) all_input_digests = [ sources_digest, inits_digest.directory_digest, resolved_requirements_pex.directory_digest, ] merged_input_files = yield Get( Digest, DirectoriesToMerge, DirectoriesToMerge(directories=tuple(all_input_digests)), ) interpreter_search_paths = create_path_env_var(python_setup.interpreter_search_paths) pex_exe_env = { 'PATH': interpreter_search_paths, **subprocess_encoding_environment.invocation_environment_dict } # NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like # `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote # execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable # somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not # necessarily the interpreter that PEX will use to execute the generated .pex file. request = ExecuteProcessRequest( argv=("python", f'./{output_pytest_requirements_pex_filename}'), env=pex_exe_env, input_files=merged_input_files, description=f'Run Pytest for {test_target.address.reference()}', ) result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, request) status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE yield TestResult( status=status, stdout=result.stdout.decode(), stderr=result.stderr.decode(), )