async def create_python_awslambda(
lambda_tgt_adaptor: PythonAWSLambdaAdaptor,
lambdex_setup: LambdexSetup,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> CreatedAWSLambda:
# TODO: We must enforce that everything is built for Linux, no matter the local platform.
pex_filename = f"{lambda_tgt_adaptor.address.target_name}.pex"
pex_request = CreatePexFromTargetClosure(
addresses=Addresses([lambda_tgt_adaptor.address]),
entry_point=None,
output_filename=pex_filename,
)
pex = await Get[Pex](CreatePexFromTargetClosure, pex_request)
merged_input_files = await Get[Digest](DirectoriesToMerge(
directories=(pex.directory_digest,
lambdex_setup.requirements_pex.directory_digest)))
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
lambdex_args = ("build", "-e", lambda_tgt_adaptor.handler, pex_filename)
process_request = lambdex_setup.requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./lambdex.pex",
pex_args=lambdex_args,
input_files=merged_input_files,
output_files=(pex_filename, ),
description=f"Run Lambdex for {lambda_tgt_adaptor.address.reference()}",
)
result = await Get[ExecuteProcessResult](ExecuteProcessRequest,
process_request)
return CreatedAWSLambda(digest=result.output_directory_digest,
name=pex_filename)
async def create_python_binary(
python_binary_adaptor: PythonBinaryAdaptor) -> CreatedBinary:
#TODO(#8420) This way of calculating the entry point works but is a bit hackish.
entry_point = None
if hasattr(python_binary_adaptor, 'entry_point'):
entry_point = python_binary_adaptor.entry_point
else:
sources_snapshot = python_binary_adaptor.sources.snapshot
if len(sources_snapshot.files) == 1:
target = await Get[HydratedTarget](Address,
python_binary_adaptor.address)
output = await Get[SourceRootStrippedSources](HydratedTarget,
target)
root_filename = output.snapshot.files[0]
entry_point = PythonBinary.translate_source_path_to_py_module_specifier(
root_filename)
request = CreatePexFromTargetClosure(
build_file_addresses=BuildFileAddresses(
(python_binary_adaptor.address, )),
entry_point=entry_point,
output_filename=f'{python_binary_adaptor.address.target_name}.pex')
pex = await Get[Pex](CreatePexFromTargetClosure, request)
return CreatedBinary(digest=pex.directory_digest,
binary_name=pex.output_filename)
async def create_python_binary(
python_binary_adaptor: PythonBinaryAdaptor) -> CreatedBinary:
# TODO(#8420) This way of calculating the entry point works but is a bit hackish.
if hasattr(python_binary_adaptor, "entry_point"):
entry_point = python_binary_adaptor.entry_point
else:
sources = await Get[SourceFiles](AllSourceFilesRequest(
[python_binary_adaptor], strip_source_roots=True))
# NB: `python_binary` targets may have 0-1 sources. This is enforced by
# `PythonBinaryAdaptor`.
if len(sources.snapshot.files) == 1:
module_name = sources.snapshot.files[0]
entry_point = PythonBinary.translate_source_path_to_py_module_specifier(
module_name)
else:
entry_point = None
request = CreatePexFromTargetClosure(
addresses=Addresses((python_binary_adaptor.address, )),
entry_point=entry_point,
output_filename=f"{python_binary_adaptor.address.target_name}.pex",
)
pex = await Get[Pex](CreatePexFromTargetClosure, request)
return CreatedBinary(digest=pex.directory_digest,
binary_name=pex.output_filename)
async def create_python_binary(fields: PythonBinaryFields) -> CreatedBinary:
entry_point: Optional[str]
if fields.entry_point.value is not None:
entry_point = fields.entry_point.value
else:
# TODO: rework determine_source_files.py to work with the Target API. It should take the
# Sources AsyncField as input, rather than TargetAdaptor.
sources_result = await Get[SourcesResult](SourcesRequest,
fields.sources.request)
stripped_sources = await Get[SourceRootStrippedSources](
StripSnapshotRequest(sources_result.snapshot))
source_files = stripped_sources.snapshot.files
# NB: `PythonBinarySources` enforces that we have 0-1 sources.
if len(source_files) == 1:
module_name = source_files[0]
entry_point = PythonBinary.translate_source_path_to_py_module_specifier(
module_name)
else:
entry_point = None
request = CreatePexFromTargetClosure(
addresses=Addresses([fields.address]),
entry_point=entry_point,
output_filename=f"{fields.address.target_name}.pex",
)
pex = await Get[Pex](CreatePexFromTargetClosure, request)
return CreatedBinary(digest=pex.directory_digest,
binary_name=pex.output_filename)
async def run_python_repl(repl: PythonRepl) -> ReplBinary:
targets = await Get[TransitiveHydratedTargets](Addresses, repl.addresses)
python_addresses = Addresses(
ht.adaptor.address for ht in targets.closure
if isinstance(ht.adaptor, PythonTargetAdaptor))
create_pex = CreatePexFromTargetClosure(
addresses=python_addresses,
output_filename="python-repl.pex",
)
repl_pex = await Get[Pex](CreatePexFromTargetClosure, create_pex)
return ReplBinary(
digest=repl_pex.directory_digest,
binary_name=repl_pex.output_filename,
)
async def run_ipython_repl(repl: IPythonRepl, ipython: IPython) -> ReplBinary:
targets = await Get[TransitiveHydratedTargets](Addresses, repl.addresses)
python_addresses = Addresses(
ht.adaptor.address for ht in targets.closure
if isinstance(ht.adaptor, PythonTargetAdaptor))
create_pex = CreatePexFromTargetClosure(
addresses=python_addresses,
output_filename="ipython-repl.pex",
entry_point=ipython.get_entry_point(),
additional_requirements=ipython.get_requirement_specs(),
)
repl_pex = await Get[Pex](CreatePexFromTargetClosure, create_pex)
return ReplBinary(
digest=repl_pex.directory_digest,
binary_name=repl_pex.output_filename,
)
async def setup_pytest_for_target(
adaptor_with_origin: PythonTestsAdaptorWithOrigin,
pytest: PyTest,
test_options: TestOptions,
python_setup: PythonSetup,
) -> TestTargetSetup:
# TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage
# configuration via #7490.
adaptor = adaptor_with_origin.adaptor
test_addresses = Addresses((adaptor.address,))
# TODO(John Sirois): PexInterpreterConstraints are gathered in the same way by the
# `create_pex_from_target_closure` rule, factor up.
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](Addresses, test_addresses)
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=all_target_adaptors, python_setup=python_setup
)
# Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints
# of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for
# CreatePex requests.
create_pex = functools.partial(CreatePex, interpreter_constraints=interpreter_constraints)
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args_for_pytest = ("--not-zip-safe",)
run_coverage = test_options.values.run_coverage
plugin_file_digest: Optional[Digest] = (
await Get[Digest](InputFilesContent, get_coverage_plugin_input()) if run_coverage else None
)
pytest_pex = await Get[Pex](
CreatePex,
create_pex(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
additional_args=additional_args_for_pytest,
input_files_digest=plugin_file_digest,
),
)
requirements_pex = await Get[Pex](
CreatePexFromTargetClosure(
addresses=test_addresses,
output_filename="requirements.pex",
include_source_files=False,
additional_args=additional_args_for_pytest,
)
)
test_runner_pex = await Get[Pex](
CreatePex,
create_pex(
output_filename="test_runner.pex",
entry_point="pytest:main",
interpreter_constraints=interpreter_constraints,
additional_args=(
"--pex-path",
":".join(
pex_request.output_filename
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any pytest transitive requirements will shadow corresponding user
# requirements which will lead to problems when APIs that are used by either
# `pytest:main` or the tests themselves break between the two versions.
for pex_request in (pytest_pex, requirements_pex)
),
),
),
)
chrooted_sources = await Get[ChrootedPythonSources](HydratedTargets(all_targets))
directories_to_merge = [
chrooted_sources.snapshot.directory_digest,
requirements_pex.directory_digest,
pytest_pex.directory_digest,
test_runner_pex.directory_digest,
]
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest([adaptor_with_origin], strip_source_roots=True)
)
specified_source_file_names = specified_source_files.snapshot.files
coverage_args = []
if run_coverage:
coveragerc = await Get[Coveragerc](
CoveragercRequest(HydratedTargets(all_targets), test_time=True)
)
directories_to_merge.append(coveragerc.digest)
packages_to_cover = get_packages_to_cover(
target=adaptor, specified_source_files=specified_source_files,
)
coverage_args = [
"--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html
]
for package in packages_to_cover:
coverage_args.extend(["--cov", package])
merged_input_files = await Get[Digest](
DirectoriesToMerge(directories=tuple(directories_to_merge))
)
timeout_seconds = calculate_timeout_seconds(
timeouts_enabled=pytest.options.timeouts,
target_timeout=getattr(adaptor, "timeout", None),
timeout_default=pytest.options.timeout_default,
timeout_maximum=pytest.options.timeout_maximum,
)
return TestTargetSetup(
test_runner_pex=test_runner_pex,
args=(*pytest.options.args, *coverage_args, *sorted(specified_source_file_names)),
input_files_digest=merged_input_files,
timeout_seconds=timeout_seconds,
)
async def setup_pytest_for_target(
test_target: PythonTestsAdaptor,
pytest: PyTest,
test_options: TestOptions,
) -> TestTargetSetup:
# TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage
# configuration via #7490.
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
resolved_requirements_pex = await Get[Pex](
CreatePexFromTargetClosure(
build_file_addresses=BuildFileAddresses((test_target.address, )),
output_filename='pytest-with-requirements.pex',
entry_point="pytest:main",
additional_requirements=pytest.get_requirement_strings(),
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args=("--not-zip-safe", ),
include_source_files=False,
))
chrooted_sources = await Get[ChrootedPythonSources](
HydratedTargets(all_targets))
directories_to_merge = [
chrooted_sources.digest,
resolved_requirements_pex.directory_digest,
]
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = await Get[
SourceRootStrippedSources](Address, test_target.address.to_address())
coverage_args = []
test_target_sources_file_names = source_root_stripped_test_target_sources.snapshot.files
if test_options.values.run_coverage:
coveragerc_digest = await Get[Digest](
InputFilesContent, get_coveragerc_input(DEFAULT_COVERAGE_CONFIG))
directories_to_merge.append(coveragerc_digest)
packages_to_cover = get_packages_to_cover(
test_target,
source_root_stripped_file_paths=test_target_sources_file_names,
)
coverage_args = [
'--cov-report=', # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html
]
for package in packages_to_cover:
coverage_args.extend(['--cov', package])
merged_input_files = await Get[Digest](
DirectoriesToMerge(directories=tuple(directories_to_merge)))
timeout_seconds = calculate_timeout_seconds(
timeouts_enabled=pytest.options.timeouts,
target_timeout=getattr(test_target, 'timeout', None),
timeout_default=pytest.options.timeout_default,
timeout_maximum=pytest.options.timeout_maximum,
)
return TestTargetSetup(
requirements_pex=resolved_requirements_pex,
args=(*pytest.options.args, *coverage_args,
*sorted(test_target_sources_file_names)),
input_files_digest=merged_input_files,
timeout_seconds=timeout_seconds,
)
async def run_python_test(
test_target: PythonTestsAdaptor, pytest: PyTest, python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment
) -> TestResult:
"""Runs pytest for one target."""
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
resolved_requirements_pex = await Get[Pex](CreatePexFromTargetClosure(
build_file_addresses=BuildFileAddresses((test_target.address, )),
output_filename=output_pytest_requirements_pex_filename,
entry_point="pytest:main",
additional_requirements=pytest.get_requirement_strings(),
include_source_files=False))
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = await Get[
SourceRootStrippedSources](Address, test_target.address.to_address())
source_root_stripped_sources = await MultiGet(
Get[SourceRootStrippedSources](HydratedTarget, hydrated_target)
for hydrated_target in all_targets)
stripped_sources_digests = tuple(
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources)
sources_digest = await Get[Digest](
DirectoriesToMerge(directories=stripped_sources_digests))
inits_digest = await Get[InjectedInitDigest](Digest, sources_digest)
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
)), )
test_target_sources_file_names = sorted(
source_root_stripped_test_target_sources.snapshot.files)
timeout_seconds = calculate_timeout_seconds(
timeouts_enabled=pytest.options.timeouts,
target_timeout=getattr(test_target, 'timeout', None),
timeout_default=pytest.options.timeout_default,
timeout_maximum=pytest.options.timeout_maximum,
)
request = resolved_requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f'./{output_pytest_requirements_pex_filename}',
pex_args=(*pytest.get_args(), *test_target_sources_file_names),
input_files=merged_input_files,
description=f'Run Pytest for {test_target.address.reference()}',
timeout_seconds=timeout_seconds
if timeout_seconds is not None else 9999)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return TestResult.from_fallible_execute_process_result(result)