def create_python_binary(python_binary_adaptor: PythonBinaryAdaptor,
python_setup: PythonSetup) -> CreatedBinary:
transitive_hydrated_targets = yield Get(
TransitiveHydratedTargets,
BuildFileAddresses((python_binary_adaptor.address, )))
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=tuple(all_targets), python_setup=python_setup)
source_root_stripped_sources = yield [
Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
for target_adaptor in all_targets
]
#TODO(#8420) This way of calculating the entry point works but is a bit hackish.
entry_point = None
if hasattr(python_binary_adaptor, 'entry_point'):
entry_point = python_binary_adaptor.entry_point
else:
sources_snapshot = python_binary_adaptor.sources.snapshot
if len(sources_snapshot.files) == 1:
target = transitive_hydrated_targets.roots[0]
output = yield Get(SourceRootStrippedSources, HydratedTarget,
target)
root_filename = output.snapshot.files[0]
entry_point = PythonBinary.translate_source_path_to_py_module_specifier(
root_filename)
stripped_sources_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources
]
sources_digest = yield Get(
Digest,
DirectoriesToMerge(directories=tuple(stripped_sources_digests)))
inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
all_input_digests = [sources_digest, inits_digest.directory_digest]
merged_input_files = yield Get(
Digest, DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)))
requirements = PexRequirements.create_from_adaptors(all_target_adaptors)
output_filename = f"{python_binary_adaptor.address.target_name}.pex"
create_requirements_pex = CreatePex(
output_filename=output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=entry_point,
input_files_digest=merged_input_files,
)
pex = yield Get(Pex, CreatePex, create_requirements_pex)
yield CreatedBinary(digest=pex.directory_digest,
binary_name=pex.output_filename)
def run_python_test(test_target, pytest, python_setup, source_root_config,
subprocess_encoding_environment):
"""Runs pytest for one target."""
# TODO(7726): replace this with a proper API to get the `closure` for a
# TransitiveHydratedTarget.
transitive_hydrated_targets = yield Get(
TransitiveHydratedTargets, BuildFileAddresses((test_target.address, )))
all_targets = [t.adaptor for t in transitive_hydrated_targets.closure]
interpreter_constraints = {
constraint
for target_adaptor in all_targets
for constraint in python_setup.compatibility_or_constraints(
getattr(target_adaptor, 'compatibility', None))
}
# Produce a pex containing pytest and all transitive 3rdparty requirements.
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
all_target_requirements = []
for maybe_python_req_lib in all_targets:
# This is a python_requirement()-like target.
if hasattr(maybe_python_req_lib, 'requirement'):
all_target_requirements.append(
str(maybe_python_req_lib.requirement))
# This is a python_requirement_library()-like target.
if hasattr(maybe_python_req_lib, 'requirements'):
for py_req in maybe_python_req_lib.requirements:
all_target_requirements.append(str(py_req.requirement))
all_requirements = all_target_requirements + list(
pytest.get_requirement_strings())
resolved_requirements_pex = yield Get(
ResolvedRequirementsPex,
ResolveRequirementsRequest(
output_filename=output_pytest_requirements_pex_filename,
requirements=tuple(sorted(all_requirements)),
interpreter_constraints=tuple(sorted(interpreter_constraints)),
entry_point="pytest:main",
))
source_roots = source_root_config.get_source_roots()
# Gather sources and adjust for the source root.
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
# TODO(7714): restore the full source name for the stdout of the Pytest run.
sources_snapshots_and_source_roots = []
for maybe_source_target in all_targets:
if hasattr(maybe_source_target, 'sources'):
tgt_snapshot = maybe_source_target.sources.snapshot
tgt_source_root = source_roots.find_by_path(
maybe_source_target.address.spec_path)
sources_snapshots_and_source_roots.append(
(tgt_snapshot, tgt_source_root))
all_sources_digests = yield [
Get(
Digest,
DirectoryWithPrefixToStrip(
directory_digest=snapshot.directory_digest,
prefix=source_root.path))
for snapshot, source_root in sources_snapshots_and_source_roots
]
sources_digest = yield Get(
Digest,
DirectoriesToMerge(directories=tuple(all_sources_digests)),
)
inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
all_input_digests = [
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
]
merged_input_files = yield Get(
Digest,
DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)),
)
interpreter_search_paths = create_path_env_var(
python_setup.interpreter_search_paths)
pex_exe_env = {
'PATH': interpreter_search_paths,
**subprocess_encoding_environment.invocation_environment_dict
}
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
# TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target
# platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735.
request = ExecuteProcessRequest(
argv=("python",
'./{}'.format(output_pytest_requirements_pex_filename)),
env=pex_exe_env,
input_files=merged_input_files,
description='Run pytest for {}'.format(
test_target.address.reference()),
)
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest,
request)
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield TestResult(
status=status,
stdout=result.stdout.decode(),
stderr=result.stderr.decode(),
)
def run_python_test(
test_target: PythonTestsAdaptor, pytest: PyTest, python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment
) -> TestResult:
"""Runs pytest for one target."""
# TODO(7726): replace this with a proper API to get the `closure` for a
# TransitiveHydratedTarget.
transitive_hydrated_targets = yield Get(
TransitiveHydratedTargets, BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = tuple(t.adaptor for t in all_targets)
interpreter_constraints = PexInterpreterContraints.create_from_adaptors(
adaptors=tuple(all_target_adaptors), python_setup=python_setup)
# Produce a pex containing pytest and all transitive 3rdparty requirements.
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=pytest.get_requirement_strings())
resolved_requirements_pex = yield Get(
Pex,
CreatePex(
output_filename=output_pytest_requirements_pex_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point="pytest:main",
))
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = yield Get(
SourceRootStrippedSources, Address, test_target.address.to_address())
source_root_stripped_sources = yield [
Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
for target_adaptor in all_targets
]
stripped_sources_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources
]
sources_digest = yield Get(
Digest,
DirectoriesToMerge(directories=tuple(stripped_sources_digests)),
)
inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
all_input_digests = [
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
]
merged_input_files = yield Get(
Digest,
DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)),
)
test_target_sources_file_names = sorted(
source_root_stripped_test_target_sources.snapshot.files)
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
request = resolved_requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f'./{output_pytest_requirements_pex_filename}',
pex_args=test_target_sources_file_names,
input_files=merged_input_files,
description=f'Run Pytest for {test_target.address.reference()}',
)
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest,
request)
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield TestResult(
status=status,
stdout=result.stdout.decode(),
stderr=result.stderr.decode(),
)
def run_python_test(test_target, pytest, python_setup, source_root_config, subprocess_encoding_environment):
"""Runs pytest for one target."""
# TODO(7726): replace this with a proper API to get the `closure` for a
# TransitiveHydratedTarget.
transitive_hydrated_targets = yield Get(
TransitiveHydratedTargets, BuildFileAddresses((test_target.address,))
)
all_targets = [t.adaptor for t in transitive_hydrated_targets.closure]
interpreter_constraints = {
constraint
for target_adaptor in all_targets
for constraint in python_setup.compatibility_or_constraints(
getattr(target_adaptor, 'compatibility', None)
)
}
# Produce a pex containing pytest and all transitive 3rdparty requirements.
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
all_target_requirements = []
for maybe_python_req_lib in all_targets:
# This is a python_requirement()-like target.
if hasattr(maybe_python_req_lib, 'requirement'):
all_target_requirements.append(str(maybe_python_req_lib.requirement))
# This is a python_requirement_library()-like target.
if hasattr(maybe_python_req_lib, 'requirements'):
for py_req in maybe_python_req_lib.requirements:
all_target_requirements.append(str(py_req.requirement))
all_requirements = all_target_requirements + list(pytest.get_requirement_strings())
resolved_requirements_pex = yield Get(
RequirementsPex, RequirementsPexRequest(
output_filename=output_pytest_requirements_pex_filename,
requirements=tuple(sorted(all_requirements)),
interpreter_constraints=tuple(sorted(interpreter_constraints)),
entry_point="pytest:main",
)
)
# Gather sources and adjust for source roots.
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of raising to
# simplify the hasattr() checks here!
source_roots = source_root_config.get_source_roots()
sources_digest_to_source_roots: Dict[Digest, Optional[SourceRoot]] = {}
for maybe_source_target in all_targets:
if not hasattr(maybe_source_target, 'sources'):
continue
digest = maybe_source_target.sources.snapshot.directory_digest
source_root = source_roots.find_by_path(maybe_source_target.address.spec_path)
if maybe_source_target.type_alias == Files.alias():
# Loose `Files`, as opposed to `Resources` or `PythonTarget`s, have no (implied) package
# structure and so we do not remove their source root like we normally do, so that Python
# filesystem APIs may still access the files. See pex_build_util.py's `_create_source_dumper`.
source_root = None
sources_digest_to_source_roots[digest] = source_root.path if source_root else ""
stripped_sources_digests = yield [
Get(Digest, DirectoryWithPrefixToStrip(directory_digest=digest, prefix=source_root))
for digest, source_root in sources_digest_to_source_roots.items()
]
sources_digest = yield Get(
Digest, DirectoriesToMerge(directories=tuple(stripped_sources_digests)),
)
inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
all_input_digests = [
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
]
merged_input_files = yield Get(
Digest,
DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)),
)
interpreter_search_paths = create_path_env_var(python_setup.interpreter_search_paths)
pex_exe_env = {
'PATH': interpreter_search_paths,
**subprocess_encoding_environment.invocation_environment_dict
}
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
request = ExecuteProcessRequest(
argv=("python", f'./{output_pytest_requirements_pex_filename}'),
env=pex_exe_env,
input_files=merged_input_files,
description=f'Run Pytest for {test_target.address.reference()}',
)
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest, request)
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield TestResult(
status=status,
stdout=result.stdout.decode(),
stderr=result.stderr.decode(),
)
async def run_python_test(
test_target: PythonTestsAdaptor, pytest: PyTest, python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment
) -> TestResult:
"""Runs pytest for one target."""
# TODO(7726): replace this with a proper API to get the `closure` for a
# TransitiveHydratedTarget.
transitive_hydrated_targets = await Get(
TransitiveHydratedTargets, BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = tuple(t.adaptor for t in all_targets)
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=tuple(all_target_adaptors), python_setup=python_setup)
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=pytest.get_requirement_strings())
resolved_requirements_pex = await Get(
Pex,
CreatePex(
output_filename=output_pytest_requirements_pex_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point="pytest:main",
))
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = await Get(
SourceRootStrippedSources, Address, test_target.address.to_address())
source_root_stripped_sources = await MultiGet(
Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
for target_adaptor in all_targets)
stripped_sources_digests = tuple(
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources)
sources_digest = await Get(
Digest, DirectoriesToMerge(directories=stripped_sources_digests))
inits_digest = await Get(InjectedInitDigest, Digest, sources_digest)
merged_input_files = await Get(
Digest,
DirectoriesToMerge(directories=(
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
)),
)
test_target_sources_file_names = sorted(
source_root_stripped_test_target_sources.snapshot.files)
request = resolved_requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f'./{output_pytest_requirements_pex_filename}',
pex_args=(*pytest.get_args(), *test_target_sources_file_names),
input_files=merged_input_files,
description=f'Run Pytest for {test_target.address.reference()}',
# TODO(#8584): hook this up to TestRunnerTaskMixin so that we can configure the default timeout
# and also use the specified max timeout time.
timeout_seconds=getattr(test_target, 'timeout', 60))
result = await Get(FallibleExecuteProcessResult, ExecuteProcessRequest,
request)
return TestResult.from_fallible_execute_process_result(result)