async def create_ipython_repl_request(
repl: IPythonRepl, ipython: IPython, pex_env: PexEnvironment
) -> ReplRequest:
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in ipython_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
Addresses(tgt.address for tgt in repl.targets), internal_only=True
),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)
)
ipython_request = Get(
Pex,
PexRequest(
output_filename="ipython.pex",
entry_point=ipython.entry_point,
requirements=PexRequirements(ipython.all_requirements),
interpreter_constraints=requirements_pex_request.interpreter_constraints,
internal_only=True,
),
)
requirements_pex, sources, ipython_pex = await MultiGet(
requirements_request, sources_request, ipython_request
)
merged_digest = await Get(
Digest,
MergeDigests(
(requirements_pex.digest, sources.source_files.snapshot.digest, ipython_pex.digest)
),
)
args = [repl.in_chroot(ipython_pex.name)]
if ipython.options.ignore_cwd:
args.append("--ignore-cwd")
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict,
"PEX_PATH": repl.in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
}
return ReplRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def create_python_repl_request(repl: PythonRepl, pex_env: PexEnvironment) -> ReplRequest:
requirements_request = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(
Addresses(tgt.address for tgt in repl.targets), internal_only=True
),
)
sources_request = Get(
PythonSourceFiles, PythonSourceFilesRequest(repl.targets, include_files=True)
)
requirements_pex, sources = await MultiGet(requirements_request, sources_request)
merged_digest = await Get(
Digest, MergeDigests((requirements_pex.digest, sources.source_files.snapshot.digest))
)
chrooted_source_roots = [repl.in_chroot(sr) for sr in sources.source_roots]
extra_env = {**pex_env.environment_dict, "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots)}
return ReplRequest(
digest=merged_digest, args=(repl.in_chroot(requirements_pex.name),), extra_env=extra_env
)
async def create_python_binary_run_request(
field_set: PythonBinaryFieldSet,
python_binary_defaults: PythonBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point = field_set.entry_point.value
if entry_point is None:
# TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
# we only need the path relative to the source root.
binary_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources))
stripped_binary_sources = await Get(
StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ())
)
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
stripped_binary_sources.snapshot.files
)
if entry_point is None:
raise InvalidFieldException(
"You must either specify `sources` or `entry_point` for the target "
f"{repr(field_set.address)} in order to run it, but both fields were undefined."
)
transitive_targets = await Get(TransitiveTargets, Addresses([field_set.address]))
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(Addresses([field_set.address]), internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True)
)
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.interpreter_constraints,
additional_args=field_set.generate_additional_args(python_binary_defaults),
internal_only=True,
),
)
requirements, sources, runner_pex = await MultiGet(
requirements_request, sources_request, runner_pex_request
)
merged_digest = await Get(
Digest,
MergeDigests(
[requirements.digest, sources.source_files.snapshot.digest, runner_pex.digest]
),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict,
"PEX_PATH": in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
}
return RunRequest(
digest=merged_digest,
args=(in_chroot(runner_pex.name), "-m", entry_point),
extra_env=extra_env,
)
async def setup_pytest_for_target(
field_set: PythonTestFieldSet,
pytest: PyTest,
test_subsystem: TestSubsystem,
python_setup: PythonSetup,
coverage_config: CoverageConfig,
coverage_subsystem: CoverageSubsystem,
) -> TestTargetSetup:
test_addresses = Addresses((field_set.address, ))
transitive_targets = await Get(TransitiveTargets, Addresses,
test_addresses)
all_targets = transitive_targets.closure
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(tgt[PythonInterpreterCompatibility] for tgt in all_targets
if tgt.has_field(PythonInterpreterCompatibility)),
python_setup,
)
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args_for_pytest = ("--not-zip-safe", )
pytest_pex_request = Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
interpreter_constraints=interpreter_constraints,
additional_args=additional_args_for_pytest,
internal_only=True,
),
)
# Defaults to zip_safe=False.
requirements_pex_request = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(test_addresses,
internal_only=True),
)
test_runner_pex_request = Get(
Pex,
PexRequest(
interpreter_constraints=interpreter_constraints,
output_filename="test_runner.pex",
entry_point="pytest:main",
additional_args=(
"--pex-path",
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any pytest transitive requirements will shadow corresponding user
# requirements which will lead to problems when APIs that are used by either
# `pytest:main` or the tests themselves break between the two versions.
":".join((
pytest_pex_request.subject.output_filename,
requirements_pex_request.subject.output_filename,
)),
),
internal_only=True,
),
)
prepared_sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets, include_files=True))
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
field_set_source_files_request = Get(
SourceFiles, SourceFilesRequest([field_set.sources]))
(
pytest_pex,
requirements_pex,
test_runner_pex,
prepared_sources,
field_set_source_files,
) = await MultiGet(
pytest_pex_request,
requirements_pex_request,
test_runner_pex_request,
prepared_sources_request,
field_set_source_files_request,
)
input_digest = await Get(
Digest,
MergeDigests((
coverage_config.digest,
prepared_sources.source_files.snapshot.digest,
requirements_pex.digest,
pytest_pex.digest,
test_runner_pex.digest,
)),
)
coverage_args = []
if test_subsystem.use_coverage:
cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else (
".", )
coverage_args = [
"--cov-report=", # Turn off output.
*itertools.chain.from_iterable(["--cov", cov_path]
for cov_path in cov_paths),
]
return TestTargetSetup(
test_runner_pex=test_runner_pex,
args=(*pytest.options.args, *coverage_args,
*field_set_source_files.files),
input_digest=input_digest,
source_roots=prepared_sources.source_roots,
timeout_seconds=field_set.timeout.calculate_from_global_options(
pytest),
xml_dir=pytest.options.junit_xml_dir,
junit_family=pytest.options.junit_family,
execution_slot_variable=pytest.options.execution_slot_var,
)
async def setup_pytest_for_target(
request: TestSetupRequest,
pytest: PyTest,
test_subsystem: TestSubsystem,
python_setup: PythonSetup,
coverage_config: CoverageConfig,
coverage_subsystem: CoverageSubsystem,
global_options: GlobalOptions,
) -> TestSetup:
test_addresses = Addresses((request.field_set.address, ))
transitive_targets = await Get(TransitiveTargets, Addresses,
test_addresses)
all_targets = transitive_targets.closure
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(tgt[PythonInterpreterCompatibility] for tgt in all_targets
if tgt.has_field(PythonInterpreterCompatibility)),
python_setup,
)
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args_for_pytest = ("--not-zip-safe", )
pytest_pex_request = Get(
Pex,
PexRequest(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
interpreter_constraints=interpreter_constraints,
additional_args=additional_args_for_pytest,
internal_only=True,
),
)
# Defaults to zip_safe=False.
requirements_pex_request = Get(
Pex,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements(test_addresses,
internal_only=True),
)
test_runner_pex_request = Get(
Pex,
PexRequest(
interpreter_constraints=interpreter_constraints,
output_filename="test_runner.pex",
entry_point="pytest:main",
additional_args=(
"--pex-path",
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any pytest transitive requirements will shadow corresponding user
# requirements which will lead to problems when APIs that are used by either
# `pytest:main` or the tests themselves break between the two versions.
":".join((
pytest_pex_request.subject.output_filename,
requirements_pex_request.subject.output_filename,
)),
),
internal_only=True,
),
)
prepared_sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(all_targets, include_files=True))
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
field_set_source_files_request = Get(
SourceFiles, SourceFilesRequest([request.field_set.sources]))
(
pytest_pex,
requirements_pex,
test_runner_pex,
prepared_sources,
field_set_source_files,
) = await MultiGet(
pytest_pex_request,
requirements_pex_request,
test_runner_pex_request,
prepared_sources_request,
field_set_source_files_request,
)
input_digest = await Get(
Digest,
MergeDigests((
coverage_config.digest,
prepared_sources.source_files.snapshot.digest,
requirements_pex.digest,
pytest_pex.digest,
test_runner_pex.digest,
)),
)
add_opts = [f"--color={'yes' if global_options.options.colors else 'no'}"]
output_files = []
results_file_name = None
if pytest.options.junit_xml_dir and not request.is_debug:
results_file_name = f"{request.field_set.address.path_safe_spec}.xml"
add_opts.extend((f"--junitxml={results_file_name}", "-o",
f"junit_family={pytest.options.junit_family}"))
output_files.append(results_file_name)
coverage_args = []
if test_subsystem.use_coverage and not request.is_debug:
output_files.append(".coverage")
cov_paths = coverage_subsystem.filter if coverage_subsystem.filter else (
".", )
coverage_args = [
"--cov-report=", # Turn off output.
*itertools.chain.from_iterable(["--cov", cov_path]
for cov_path in cov_paths),
]
extra_env = {
"PYTEST_ADDOPTS": " ".join(add_opts),
"PEX_EXTRA_SYS_PATH": ":".join(prepared_sources.source_roots),
}
if test_subsystem.force and not request.is_debug:
# This is a slightly hacky way to force the process to run: since the env var
# value is unique, this input combination will never have been seen before,
# and therefore never cached. The two downsides are:
# 1. This leaks into the test's environment, albeit with a funky var name that is
# unlikely to cause problems in practice.
# 2. This run will be cached even though it can never be re-used.
# TODO: A more principled way of forcing rules to run?
uuid = await Get(UUID, UUIDRequest())
extra_env["__PANTS_FORCE_TEST_RUN__"] = str(uuid)
process = await Get(
Process,
PexProcess(
test_runner_pex,
argv=(*pytest.options.args, *coverage_args,
*field_set_source_files.files),
extra_env=extra_env,
input_digest=input_digest,
output_files=output_files,
timeout_seconds=request.field_set.timeout.
calculate_from_global_options(pytest),
execution_slot_variable=pytest.options.execution_slot_var,
description=f"Run Pytest for {request.field_set.address}",
level=LogLevel.DEBUG,
),
)
return TestSetup(process, results_file_name=results_file_name)