async def run_setup_py(
req: RunSetupPyRequest,
setuptools_setup: SetuptoolsSetup,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment
) -> RunSetupPyResult:
"""Run a setup.py command on a single exported target."""
merged_input_files = await Get[Digest](
DirectoriesToMerge(directories=(
req.chroot.digest,
setuptools_setup.requirements_pex.directory_digest))
)
# The setuptools dist dir, created by it under the chroot (not to be confused with
# pants's own dist dir, at the buildroot).
# TODO: The user can change this with the --dist-dir flag to the sdist and bdist_wheel commands.
# See https://github.com/pantsbuild/pants/issues/8912.
dist_dir = 'dist/'
request = setuptools_setup.requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./setuptools.pex",
pex_args=('setup.py', *req.args),
input_files=merged_input_files,
# setuptools commands that create dists write them to the distdir.
# TODO: Could there be other useful files to capture?
output_directories=(dist_dir,),
description=f'Run setuptools for {req.exported_target.hydrated_target.address.reference()}',
)
result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request)
output_digest = await Get[Digest](
DirectoryWithPrefixToStrip(result.output_directory_digest, dist_dir))
return RunSetupPyResult(output_digest)
async def run_setup_py(
req: RunSetupPyRequest,
setuptools_setup: SetuptoolsSetup,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> RunSetupPyResult:
"""Run a setup.py command on a single exported target."""
merged_input_files = await Get[Digest](DirectoriesToMerge(
directories=(req.chroot.digest,
setuptools_setup.requirements_pex.directory_digest)))
# The setuptools dist dir, created by it under the chroot (not to be confused with
# pants's own dist dir, at the buildroot).
dist_dir = "dist/"
process = setuptools_setup.requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./setuptools.pex",
pex_args=("setup.py", *req.args),
input_files=merged_input_files,
# setuptools commands that create dists write them to the distdir.
# TODO: Could there be other useful files to capture?
output_directories=(dist_dir, ),
description=
f"Run setuptools for {req.exported_target.target.address.reference()}",
)
result = await Get[ProcessResult](Process, process)
output_digest = await Get[Digest](DirectoryWithPrefixToStrip(
result.output_directory_digest, dist_dir))
return RunSetupPyResult(output_digest)
async def strip_source_root(
hydrated_target: HydratedTarget,
source_root_config: SourceRootConfig) -> SourceRootStrippedSources:
"""Relativize targets to their source root, e.g.
`src/python/pants/util/strutil.py` -> `pants/util/strutil.py ."""
target_adaptor = hydrated_target.adaptor
source_roots = source_root_config.get_source_roots()
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
if not hasattr(target_adaptor, 'sources'):
return SourceRootStrippedSources(snapshot=EMPTY_SNAPSHOT)
digest = target_adaptor.sources.snapshot.directory_digest
source_root = source_roots.find_by_path(target_adaptor.address.spec_path)
if source_root is None:
# If we found no source root, use the target's dir.
# Note that when --source-unmatched is 'create' (the default) we'll never return None,
# but will return the target's dir. This check allows this code to work even if
# --source-unmatched is 'fail'.
source_root_path = target_adaptor.address.spec_path
else:
source_root_path = source_root.path
# Loose `Files`, as opposed to `Resources` or `Target`s, have no (implied) package
# structure and so we do not remove their source root like we normally do, so that filesystem
# APIs may still access the files. See pex_build_util.py's `_create_source_dumper`.
if target_adaptor.type_alias == Files.alias():
source_root_path = ''
resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=digest, prefix=source_root_path))
resulting_snapshot = await Get[Snapshot](Digest, resulting_digest)
return SourceRootStrippedSources(snapshot=resulting_snapshot)
async def strip_source_root(
hydrated_target: HydratedTarget,
source_root_config: SourceRootConfig) -> SourceRootStrippedSources:
"""Relativize targets to their source root, e.g.
`src/python/pants/util/strutil.py` -> `pants/util/strutil.py ."""
target_adaptor = hydrated_target.adaptor
source_roots = source_root_config.get_source_roots()
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
if not hasattr(target_adaptor, 'sources'):
return SourceRootStrippedSources(snapshot=EMPTY_SNAPSHOT)
digest = target_adaptor.sources.snapshot.directory_digest
source_root = source_roots.find_by_path(target_adaptor.address.spec_path)
# Loose `Files`, as opposed to `Resources` or `Target`s, have no (implied) package
# structure and so we do not remove their source root like we normally do, so that filesystem
# APIs may still access the files. See pex_build_util.py's `_create_source_dumper`.
if target_adaptor.type_alias == Files.alias():
source_root = None
resulting_digest = await Get(
Digest,
DirectoryWithPrefixToStrip(
directory_digest=digest,
prefix=source_root.path if source_root else ""))
resulting_snapshot = await Get(Snapshot, Digest, resulting_digest)
return SourceRootStrippedSources(snapshot=resulting_snapshot)
async def strip_source_roots_from_snapshot(
request: StripSnapshotRequest,
source_root_config: SourceRootConfig,
) -> SourceRootStrippedSources:
"""Removes source roots from a snapshot, e.g. `src/python/pants/util/strutil.py` ->
`pants/util/strutil.py`."""
source_roots_object = source_root_config.get_source_roots()
def determine_source_root(path: str) -> str:
source_root = source_roots_object.safe_find_by_path(path)
if source_root is not None:
return cast(str, source_root.path)
if source_root_config.options.unmatched == "fail":
raise NoSourceRootError(
f"Could not find a source root for `{path}`.")
# Otherwise, create a source root by using the parent directory.
return PurePath(path).parent.as_posix()
if request.representative_path is not None:
resulting_digest = await Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=request.snapshot.directory_digest,
prefix=determine_source_root(request.representative_path),
))
resulting_snapshot = await Get[Snapshot](Digest, resulting_digest)
return SourceRootStrippedSources(snapshot=resulting_snapshot)
files_grouped_by_source_root = {
source_root: tuple(files)
for source_root, files in itertools.groupby(request.snapshot.files,
key=determine_source_root)
}
snapshot_subsets = await MultiGet(Get[Snapshot](SnapshotSubset(
directory_digest=request.snapshot.directory_digest,
globs=PathGlobs(files),
)) for files in files_grouped_by_source_root.values())
resulting_digests = await MultiGet(
Get[Digest](DirectoryWithPrefixToStrip(
directory_digest=snapshot.directory_digest, prefix=source_root))
for snapshot, source_root in zip(snapshot_subsets,
files_grouped_by_source_root.keys()))
merged_result = await Get[Digest](DirectoriesToMerge(resulting_digests))
resulting_snapshot = await Get[Snapshot](Digest, merged_result)
return SourceRootStrippedSources(resulting_snapshot)
async def get_ancestor_init_py(
targets: Targets, source_root_config: SourceRootConfig
) -> AncestorInitPyFiles:
"""Find any ancestor __init__.py files for the given targets.
Includes sibling __init__.py files. Returns the files stripped of their source roots.
"""
source_roots = source_root_config.get_source_roots()
sources = await Get[SourceFiles](
AllSourceFilesRequest(tgt[PythonSources] for tgt in targets if tgt.has_field(PythonSources))
)
# Find the ancestors of all dirs containing .py files, including those dirs themselves.
source_dir_ancestors: Set[Tuple[str, str]] = set() # Items are (src_root, path incl. src_root).
for fp in sources.snapshot.files:
source_dir_ancestor = os.path.dirname(fp)
source_root = source_root_or_raise(source_roots, fp)
# Do not allow the repository root to leak (i.e., '.' should not be a package in setup.py).
while source_dir_ancestor != source_root:
source_dir_ancestors.add((source_root, source_dir_ancestor))
source_dir_ancestor = os.path.dirname(source_dir_ancestor)
source_dir_ancestors_list = list(source_dir_ancestors) # To force a consistent order.
# Note that we must MultiGet single globs instead of a a single Get for all the globs, because
# we match each result to its originating glob (see use of zip below).
ancestor_init_py_snapshots = await MultiGet[Snapshot](
Get[Snapshot](PathGlobs, PathGlobs([os.path.join(source_dir_ancestor[1], "__init__.py")]))
for source_dir_ancestor in source_dir_ancestors_list
)
source_root_stripped_ancestor_init_pys = await MultiGet[Digest](
Get[Digest](
DirectoryWithPrefixToStrip(
directory_digest=snapshot.directory_digest, prefix=source_dir_ancestor[0]
)
)
for snapshot, source_dir_ancestor in zip(
ancestor_init_py_snapshots, source_dir_ancestors_list
)
)
return AncestorInitPyFiles(source_root_stripped_ancestor_init_pys)
def test_strip_prefix(self):
# Set up files:
relevant_files = (
'characters/dark_tower/roland',
'characters/dark_tower/susannah',
)
all_files = (
'books/dark_tower/gunslinger',
'characters/altered_carbon/kovacs',
) + relevant_files + (
'index',
)
with temporary_dir() as temp_dir:
safe_file_dump(os.path.join(temp_dir, 'index'), 'books\ncharacters\n')
safe_file_dump(
os.path.join(temp_dir, "characters", "altered_carbon", "kovacs"),
"Envoy",
makedirs=True,
)
tower_dir = os.path.join(temp_dir, "characters", "dark_tower")
safe_file_dump(os.path.join(tower_dir, "roland"), "European Burmese", makedirs=True)
safe_file_dump(os.path.join(tower_dir, "susannah"), "Not sure actually", makedirs=True)
safe_file_dump(
os.path.join(temp_dir, "books", "dark_tower", "gunslinger"),
"1982",
makedirs=True,
)
snapshot, snapshot_with_extra_files = self.scheduler.capture_snapshots((
PathGlobsAndRoot(PathGlobs(("characters/dark_tower/*",)), temp_dir),
PathGlobsAndRoot(PathGlobs(("**",)), temp_dir),
))
# Check that we got the full snapshots that we expect
self.assertEquals(snapshot.files, relevant_files)
self.assertEquals(snapshot_with_extra_files.files, all_files)
# Strip empty prefix:
zero_prefix_stripped_digest = assert_single_element(self.scheduler.product_request(
Digest,
[DirectoryWithPrefixToStrip(snapshot.directory_digest, "")],
))
self.assertEquals(snapshot.directory_digest, zero_prefix_stripped_digest)
# Strip a non-empty prefix shared by all files:
stripped_digest = assert_single_element(self.scheduler.product_request(
Digest,
[DirectoryWithPrefixToStrip(snapshot.directory_digest, "characters/dark_tower")],
))
self.assertEquals(
stripped_digest,
Digest(
fingerprint='71e788fc25783c424db555477071f5e476d942fc958a5d06ffc1ed223f779a8c',
serialized_bytes_length=162,
)
)
expected_snapshot = assert_single_element(self.scheduler.capture_snapshots((
PathGlobsAndRoot(PathGlobs(("*",)), tower_dir),
)))
self.assertEquals(expected_snapshot.files, ('roland', 'susannah'))
self.assertEquals(stripped_digest, expected_snapshot.directory_digest)
# Try to strip a prefix which isn't shared by all files:
with self.assertRaisesWithMessageContaining(Exception, "Cannot strip prefix characters/dark_tower from root directory Digest(Fingerprint<28c47f77867f0c8d577d2ada2f06b03fc8e5ef2d780e8942713b26c5e3f434b8>, 243) - root directory contained non-matching directory named: books and file named: index"):
self.scheduler.product_request(
Digest,
[DirectoryWithPrefixToStrip(snapshot_with_extra_files.directory_digest, "characters/dark_tower")]
)
def run_python_test(test_target, pytest, python_setup, source_root_config,
subprocess_encoding_environment):
"""Runs pytest for one target."""
# TODO(7726): replace this with a proper API to get the `closure` for a
# TransitiveHydratedTarget.
transitive_hydrated_targets = yield Get(
TransitiveHydratedTargets, BuildFileAddresses((test_target.address, )))
all_targets = [t.adaptor for t in transitive_hydrated_targets.closure]
interpreter_constraints = {
constraint
for target_adaptor in all_targets
for constraint in python_setup.compatibility_or_constraints(
getattr(target_adaptor, 'compatibility', None))
}
# Produce a pex containing pytest and all transitive 3rdparty requirements.
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
all_target_requirements = []
for maybe_python_req_lib in all_targets:
# This is a python_requirement()-like target.
if hasattr(maybe_python_req_lib, 'requirement'):
all_target_requirements.append(
str(maybe_python_req_lib.requirement))
# This is a python_requirement_library()-like target.
if hasattr(maybe_python_req_lib, 'requirements'):
for py_req in maybe_python_req_lib.requirements:
all_target_requirements.append(str(py_req.requirement))
all_requirements = all_target_requirements + list(
pytest.get_requirement_strings())
resolved_requirements_pex = yield Get(
ResolvedRequirementsPex,
ResolveRequirementsRequest(
output_filename=output_pytest_requirements_pex_filename,
requirements=tuple(sorted(all_requirements)),
interpreter_constraints=tuple(sorted(interpreter_constraints)),
entry_point="pytest:main",
))
source_roots = source_root_config.get_source_roots()
# Gather sources and adjust for the source root.
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
# TODO(7714): restore the full source name for the stdout of the Pytest run.
sources_snapshots_and_source_roots = []
for maybe_source_target in all_targets:
if hasattr(maybe_source_target, 'sources'):
tgt_snapshot = maybe_source_target.sources.snapshot
tgt_source_root = source_roots.find_by_path(
maybe_source_target.address.spec_path)
sources_snapshots_and_source_roots.append(
(tgt_snapshot, tgt_source_root))
all_sources_digests = yield [
Get(
Digest,
DirectoryWithPrefixToStrip(
directory_digest=snapshot.directory_digest,
prefix=source_root.path))
for snapshot, source_root in sources_snapshots_and_source_roots
]
sources_digest = yield Get(
Digest,
DirectoriesToMerge(directories=tuple(all_sources_digests)),
)
inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
all_input_digests = [
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
]
merged_input_files = yield Get(
Digest,
DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)),
)
interpreter_search_paths = create_path_env_var(
python_setup.interpreter_search_paths)
pex_exe_env = {
'PATH': interpreter_search_paths,
**subprocess_encoding_environment.invocation_environment_dict
}
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
# TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target
# platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735.
request = ExecuteProcessRequest(
argv=("python",
'./{}'.format(output_pytest_requirements_pex_filename)),
env=pex_exe_env,
input_files=merged_input_files,
description='Run pytest for {}'.format(
test_target.address.reference()),
)
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest,
request)
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield TestResult(
status=status,
stdout=result.stdout.decode(),
stderr=result.stderr.decode(),
)
def run_python_test(test_target, pytest, python_setup, source_root_config, subprocess_encoding_environment):
"""Runs pytest for one target."""
# TODO(7726): replace this with a proper API to get the `closure` for a
# TransitiveHydratedTarget.
transitive_hydrated_targets = yield Get(
TransitiveHydratedTargets, BuildFileAddresses((test_target.address,))
)
all_targets = [t.adaptor for t in transitive_hydrated_targets.closure]
interpreter_constraints = {
constraint
for target_adaptor in all_targets
for constraint in python_setup.compatibility_or_constraints(
getattr(target_adaptor, 'compatibility', None)
)
}
# Produce a pex containing pytest and all transitive 3rdparty requirements.
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
all_target_requirements = []
for maybe_python_req_lib in all_targets:
# This is a python_requirement()-like target.
if hasattr(maybe_python_req_lib, 'requirement'):
all_target_requirements.append(str(maybe_python_req_lib.requirement))
# This is a python_requirement_library()-like target.
if hasattr(maybe_python_req_lib, 'requirements'):
for py_req in maybe_python_req_lib.requirements:
all_target_requirements.append(str(py_req.requirement))
all_requirements = all_target_requirements + list(pytest.get_requirement_strings())
resolved_requirements_pex = yield Get(
RequirementsPex, RequirementsPexRequest(
output_filename=output_pytest_requirements_pex_filename,
requirements=tuple(sorted(all_requirements)),
interpreter_constraints=tuple(sorted(interpreter_constraints)),
entry_point="pytest:main",
)
)
# Gather sources and adjust for source roots.
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
source_roots = source_root_config.get_source_roots()
sources_digest_to_source_roots: Dict[Digest, Optional[SourceRoot]] = {}
for maybe_source_target in all_targets:
if not hasattr(maybe_source_target, 'sources'):
continue
digest = maybe_source_target.sources.snapshot.directory_digest
source_root = source_roots.find_by_path(maybe_source_target.address.spec_path)
if maybe_source_target.type_alias == Files.alias():
# Loose `Files`, as opposed to `Resources` or `PythonTarget`s, have no (implied) package
# structure and so we do not remove their source root like we normally do, so that Python
# filesystem APIs may still access the files. See pex_build_util.py's `_create_source_dumper`.
source_root = None
sources_digest_to_source_roots[digest] = source_root.path if source_root else ""
stripped_sources_digests = yield [
Get(Digest, DirectoryWithPrefixToStrip(directory_digest=digest, prefix=source_root))
for digest, source_root in sources_digest_to_source_roots.items()
]
sources_digest = yield Get(
Digest, DirectoriesToMerge(directories=tuple(stripped_sources_digests)),
)
inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
all_input_digests = [
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
]
merged_input_files = yield Get(
Digest,
DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)),
)
interpreter_search_paths = create_path_env_var(python_setup.interpreter_search_paths)
pex_exe_env = {
'PATH': interpreter_search_paths,
**subprocess_encoding_environment.invocation_environment_dict
}
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
request = ExecuteProcessRequest(
argv=("python", f'./{output_pytest_requirements_pex_filename}'),
env=pex_exe_env,
input_files=merged_input_files,
description=f'Run Pytest for {test_target.address.reference()}',
)
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, request)
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield TestResult(
status=status,
stdout=result.stdout.decode(),
stderr=result.stderr.decode(),
)
