async def merge_coverage_data(
data_batch: PytestCoverageDataBatch,
transitive_targets: TransitiveHydratedTargets,
python_setup: PythonSetup,
coverage_setup: CoverageSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> MergedCoverageData:
"""Takes all python test results and merges their coverage data into a single sql file."""
# We start with a bunch of test results, each of which has a coverage data file called `.coverage`
# We prefix each of these with their address so that we can write them all into a single pex.
coverage_directory_digests = await MultiGet(
Get[Digest](
DirectoryWithPrefixToAdd(
directory_digest=result.test_result.coverage_data.
digest, # type: ignore[attr-defined]
prefix=result.address.path_safe_spec,
)) for result in data_batch.addresses_and_test_results
if result.test_result is not None
and result.test_result.coverage_data is not None)
sources = await Get[SourceFiles](AllSourceFilesRequest(
(ht.adaptor for ht in transitive_targets.closure),
strip_source_roots=False))
sources_with_inits_snapshot = await Get[InitInjectedSnapshot](
InjectInitRequest(sources.snapshot))
coveragerc = await Get[Coveragerc](CoveragercRequest(HydratedTargets(
transitive_targets.closure),
test_time=True))
merged_input_files: Digest = await Get(
Digest,
DirectoriesToMerge(directories=(
*coverage_directory_digests,
sources_with_inits_snapshot.snapshot.directory_digest,
coveragerc.digest,
coverage_setup.requirements_pex.directory_digest,
)),
)
prefixes = [
f"{result.address.path_safe_spec}/.coverage"
for result in data_batch.addresses_and_test_results
]
coverage_args = ["combine", *prefixes]
request = coverage_setup.requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./{coverage_setup.requirements_pex.output_filename}",
pex_args=coverage_args,
input_files=merged_input_files,
output_files=(".coverage", ),
description=f"Merge coverage reports.",
)
result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request)
return MergedCoverageData(coverage_data=result.output_directory_digest)
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot:
if request.py2:
# TODO: Implement Python 2 support. This will involve, among other things: merging ancestor
# __init__.py files into the chroot, detecting packages based on the presence of __init__.py,
# and inspecting all __init__.py files for the namespace package incantation.
raise UnsupportedPythonVersion('Running setup.py commands not supported for Python 2.')
owned_deps = await Get[OwnedDependencies](DependencyOwner(request.exported_target))
targets = HydratedTargets(od.hydrated_target for od in owned_deps)
sources = await Get[SetupPySources](SetupPySourcesRequest(targets))
requirements = await Get[ExportedTargetRequirements](DependencyOwner(request.exported_target))
# Nest the sources under the src/ prefix.
src_digest = await Get[Digest](DirectoryWithPrefixToAdd(sources.digest, CHROOT_SOURCE_ROOT))
# Generate the kwargs to the setup() call.
setup_kwargs = request.exported_target.hydrated_target.adaptor.provides.setup_py_keywords.copy()
setup_kwargs.update({
'package_dir': {'': CHROOT_SOURCE_ROOT},
'packages': sources.packages,
'namespace_packages': sources.namespace_packages,
'package_data': dict(sources.package_data),
'install_requires': requirements.requirement_strs
})
ht = request.exported_target.hydrated_target
key_to_binary_spec = getattr(ht.adaptor.provides, 'binaries', {})
keys = list(key_to_binary_spec.keys())
binaries = await MultiGet(Get[HydratedTarget](
Address, Address.parse(key_to_binary_spec[key], relative_to=ht.address.spec_path))
for key in keys)
for key, binary in zip(keys, binaries):
if (not isinstance(binary.adaptor, PythonBinaryAdaptor) or
getattr(binary.adaptor, 'entry_point', None) is None):
raise InvalidEntryPoint(
f'The binary {key} exported by {ht.address.reference()} is not a valid entry point.')
entry_points = setup_kwargs['entry_points'] = setup_kwargs.get('entry_points', {})
console_scripts = entry_points['console_scripts'] = entry_points.get('console_scripts', [])
console_scripts.append(f'{key}={binary.adaptor.entry_point}')
# Generate the setup script.
setup_py_content = SETUP_BOILERPLATE.format(
target_address_spec=ht.address.reference(),
setup_kwargs_str=distutils_repr(setup_kwargs)
).encode()
extra_files_digest = await Get[Digest](
InputFilesContent([
FileContent('setup.py', setup_py_content),
FileContent('MANIFEST.in', 'include *.py'.encode()) # Make sure setup.py is included.
]))
chroot_digest = await Get[Digest](DirectoriesToMerge((src_digest, extra_files_digest)))
return SetupPyChroot(chroot_digest, json.dumps(setup_kwargs, sort_keys=True))
async def generate_coverage_report(
transitive_targets: TransitiveHydratedTargets,
python_setup: PythonSetup,
coverage_setup: CoverageSetup,
merged_coverage_data: MergedCoverageData,
coverage_toolbase: PytestCoverage,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> CoverageReport:
"""Takes all python test results and generates a single coverage report."""
requirements_pex = coverage_setup.requirements_pex
# TODO(#4535) We need a better way to do this kind of check that covers synthetic targets and rules extensibility.
python_targets = [
target for target in transitive_targets.closure
if target.adaptor.type_alias in ("python_library", "python_tests")
]
coveragerc = await Get[Coveragerc](CoveragercRequest(
HydratedTargets(python_targets)))
sources = await Get[SourceFiles](AllSourceFilesRequest(
(ht.adaptor for ht in transitive_targets.closure),
strip_source_roots=False))
sources_with_inits_snapshot = await Get[InitInjectedSnapshot](
InjectInitRequest(sources.snapshot))
merged_input_files: Digest = await Get(
Digest,
DirectoriesToMerge(directories=(
merged_coverage_data.coverage_data,
coveragerc.digest,
requirements_pex.directory_digest,
sources_with_inits_snapshot.snapshot.directory_digest,
)),
)
report_type = coverage_toolbase.options.report
coverage_args = [report_type.report_name]
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./{coverage_setup.requirements_pex.output_filename}",
pex_args=coverage_args,
input_files=merged_input_files,
output_directories=("htmlcov", ),
output_files=("coverage.xml", ),
description=f"Generate coverage report.",
)
result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request)
if report_type == ReportType.CONSOLE:
return ConsoleCoverageReport(result.stdout.decode())
return FilesystemCoverageReport(
result.output_directory_digest,
coverage_toolbase.options.report_output_path)
def assert_ancestor_init_py(self, expected_init_pys: Iterable[str],
addrs: Iterable[str]) -> None:
ancestor_init_py_files = self.request_single_product(
AncestorInitPyFiles,
Params(HydratedTargets([self.tgt(addr) for addr in addrs]),
SourceRootConfig.global_instance()))
snapshots = [
self.request_single_product(Snapshot, Params(digest))
for digest in ancestor_init_py_files.digests
]
init_py_files_found = set(
[file for snapshot in snapshots for file in snapshot.files])
# NB: Doesn't include the root __init__.py or the missing src/python/foo/bar/__init__.py.
assert sorted(expected_init_pys) == sorted(init_py_files_found)
def assert_sources(self, expected_files, expected_packages,
expected_namespace_packages, expected_package_data,
addrs):
srcs = self.request_single_product(
SetupPySources,
Params(
SetupPySourcesRequest(
HydratedTargets([self.tgt(addr) for addr in addrs])),
SourceRootConfig.global_instance()))
chroot_snapshot = self.request_single_product(Snapshot,
Params(srcs.digest))
assert sorted(expected_files) == sorted(chroot_snapshot.files)
assert sorted(expected_packages) == sorted(srcs.packages)
assert sorted(expected_namespace_packages) == sorted(
srcs.namespace_packages)
assert expected_package_data == dict(srcs.package_data)
def run_lint_rule(
*,
targets: List[HydratedTarget],
mock_linter: Callable[[PythonTargetAdaptor], LintResult],
) -> Tuple[Lint, MockConsole]:
console = MockConsole(use_colors=False)
result: Lint = run_rule(
lint,
rule_args=[
console,
HydratedTargets(targets),
UnionMembership(union_rules={TargetWithSources: [PythonTargetAdaptor]})
],
mock_gets=[
MockGet(product_type=LintResult, subject_type=PythonTargetAdaptor, mock=mock_linter),
],
)
return result, console
def run_fmt_rule(
*,
targets: List[HydratedTarget],
mock_formatter: Callable[[PythonTargetAdaptor], FmtResult],
) -> Tuple[Fmt, MockConsole]:
console = MockConsole(use_colors=False)
result: Fmt = run_rule(
fmt,
rule_args=[
console,
HydratedTargets(targets),
UnionMembership(union_rules={TargetWithSources: [PythonTargetAdaptor]})
],
mock_gets=[
MockGet(product_type=FmtResult, subject_type=PythonTargetAdaptor, mock=mock_formatter),
MockGet(product_type=FilesContent, subject_type=Digest, mock=lambda _: FilesContent([]))
],
)
return result, console
async def legacy_pex_from_targets(request: LegacyPexFromTargetsRequest,
python_setup: PythonSetup) -> PexRequest:
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
Addresses, request.addresses)
all_targets = transitive_hydrated_targets.closure
python_targets = [
t for t in all_targets if isinstance(t.adaptor, PythonTargetAdaptor)
]
resource_targets = [
t for t in all_targets
if isinstance(t.adaptor, (FilesAdaptor, ResourcesAdaptor))
]
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=all_target_adaptors, python_setup=python_setup)
input_digests = []
if request.additional_input_files:
input_digests.append(request.additional_input_files)
if request.include_source_files:
prepared_sources = await Get[ImportablePythonSources](
HydratedTargets(python_targets + resource_targets))
input_digests.append(prepared_sources.snapshot.directory_digest)
merged_input_digest = await Get[Digest](
DirectoriesToMerge(directories=tuple(input_digests)))
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=request.additional_requirements)
return PexRequest(
output_filename=request.output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=request.entry_point,
input_files_digest=merged_input_digest,
additional_args=request.additional_args,
)
def run_lint_rule(
*,
targets: List[HydratedTarget],
mock_linter: Optional[Callable[[PythonTargetAdaptor], LintResult]] = None,
) -> Tuple[Lint, str]:
if mock_linter is None:
mock_linter = lambda target_adaptor: LintResult(
exit_code=1, stdout=f"Linted the target `{target_adaptor.name}`", stderr=""
)
console = MockConsole(use_colors=False)
result: Lint = run_rule(
lint,
rule_args=[
console,
HydratedTargets(targets),
UnionMembership(union_rules={TargetWithSources: [PythonTargetAdaptor]})
],
mock_gets=[
MockGet(product_type=LintResult, subject_type=PythonTargetAdaptor, mock=mock_linter),
],
)
return result, console.stdout.getvalue()
async def setup_pytest_for_target(
config: PythonTestConfiguration,
pytest: PyTest,
test_options: TestOptions,
python_setup: PythonSetup,
) -> TestTargetSetup:
# TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage
# configuration via #7490.
test_addresses = Addresses((config.address,))
# TODO(John Sirois): PexInterpreterConstraints are gathered in the same way by the
# `create_pex_from_target_closure` rule, factor up.
transitive_targets = await Get[TransitiveTargets](Addresses, test_addresses)
all_targets = transitive_targets.closure
# TODO: factor this up? It's mostly duplicated with pex_from_targets.py.
python_targets = []
resource_targets = []
for tgt in all_targets:
if tgt.has_field(PythonSources):
python_targets.append(tgt)
# NB: PythonRequirementsFileSources is a subclass of FilesSources. We filter it out so that
# requirements.txt is not included in the PEX and so that irrelevant changes to it (e.g.
# whitespace changes) do not invalidate the PEX.
if tgt.has_field(ResourcesSources) or (
tgt.has_field(FilesSources) and not tgt.has_field(PythonRequirementsFileSources)
):
resource_targets.append(tgt)
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(tgt.get(PythonInterpreterCompatibility) for tgt in python_targets), python_setup
)
# Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints
# of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for
# CreatePex requests.
pex_request = functools.partial(PexRequest, interpreter_constraints=interpreter_constraints)
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args_for_pytest = ("--not-zip-safe",)
run_coverage = test_options.values.run_coverage
plugin_file_digest: Optional[Digest] = (
await Get[Digest](InputFilesContent, COVERAGE_PLUGIN_INPUT) if run_coverage else None
)
pytest_pex_request = pex_request(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
additional_args=additional_args_for_pytest,
sources=plugin_file_digest,
)
requirements_pex_request = LegacyPexFromTargetsRequest(
addresses=test_addresses,
output_filename="requirements.pex",
include_source_files=False,
additional_args=additional_args_for_pytest,
)
test_runner_pex_request = pex_request(
output_filename="test_runner.pex",
entry_point="pytest:main",
interpreter_constraints=interpreter_constraints,
additional_args=(
"--pex-path",
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any pytest transitive requirements will shadow corresponding user
# requirements which will lead to problems when APIs that are used by either
# `pytest:main` or the tests themselves break between the two versions.
":".join(
(pytest_pex_request.output_filename, requirements_pex_request.output_filename)
),
),
)
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
specified_source_files_request = SpecifiedSourceFilesRequest(
[(config.sources, config.origin)], strip_source_roots=True
)
# TODO(John Sirois): Support exploiting concurrency better:
# https://github.com/pantsbuild/pants/issues/9294
# Some awkward code follows in order to execute 5-6 items concurrently given the current state
# of MultiGet typing / API. Improve this since we should encourage full concurrency in general.
requests: List[Get[Any]] = [
Get[Pex](PexRequest, pytest_pex_request),
Get[Pex](LegacyPexFromTargetsRequest, requirements_pex_request),
Get[Pex](PexRequest, test_runner_pex_request),
Get[ImportablePythonSources](Targets(python_targets + resource_targets)),
Get[SourceFiles](SpecifiedSourceFilesRequest, specified_source_files_request),
]
if run_coverage:
# TODO: update coverage to use the Target API. Also, add tests.
hydrated_python_targets = await Get[HydratedTargets](
Addresses(tgt.address for tgt in python_targets)
)
requests.append(
Get[Coveragerc](
CoveragercRequest(HydratedTargets(hydrated_python_targets), test_time=True)
),
)
(
pytest_pex,
requirements_pex,
test_runner_pex,
prepared_sources,
specified_source_files,
*rest,
) = cast(
Union[
Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles],
Tuple[Pex, Pex, Pex, ImportablePythonSources, SourceFiles, Coveragerc],
],
await MultiGet(requests),
)
directories_to_merge = [
prepared_sources.snapshot.directory_digest,
requirements_pex.directory_digest,
pytest_pex.directory_digest,
test_runner_pex.directory_digest,
]
if run_coverage:
coveragerc = rest[0]
directories_to_merge.append(coveragerc.digest)
merged_input_files = await Get[Digest](
DirectoriesToMerge(directories=tuple(directories_to_merge))
)
coverage_args = []
if run_coverage:
coverage_args = [
"--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html
]
for package in config.coverage.determine_packages_to_cover(
specified_source_files=specified_source_files
):
coverage_args.extend(["--cov", package])
specified_source_file_names = sorted(specified_source_files.snapshot.files)
return TestTargetSetup(
test_runner_pex=test_runner_pex,
args=(*pytest.options.args, *coverage_args, *specified_source_file_names),
input_files_digest=merged_input_files,
timeout_seconds=config.timeout.calculate_from_global_options(pytest),
)
async def lint(
linter: PylintLinter,
pylint: Pylint,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if pylint.options.skip:
return LintResult.noop()
adaptors_with_origins = linter.adaptors_with_origins
# Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it
# doesn't lint those direct dependencies nor does it care about transitive dependencies.
hydrated_targets = [
HydratedTarget(adaptor_with_origin.adaptor)
for adaptor_with_origin in adaptors_with_origins
]
dependencies = await MultiGet(
Get[HydratedTarget](Address, dependency)
for dependency in itertools.chain.from_iterable(
ht.adaptor.dependencies for ht in hydrated_targets))
chrooted_python_sources = await Get[ImportablePythonSources](
HydratedTargets([*hydrated_targets, *dependencies]))
# NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
(adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins),
python_setup=python_setup,
)
requirements_pex = await Get[Pex](PexRequest(
output_filename="pylint.pex",
requirements=PexRequirements(pylint.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=pylint.get_entry_point(),
))
config_path: Optional[str] = pylint.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
requirements_pex.directory_digest,
config_snapshot.directory_digest,
chrooted_python_sources.snapshot.directory_digest,
)), )
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins,
strip_source_roots=True))
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./pylint.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
pylint=pylint),
input_files=merged_input_files,
description=f"Run Pylint for {address_references}",
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return LintResult.from_fallible_execute_process_result(result)
async def setup_pytest_for_target(
adaptor_with_origin: PythonTestsAdaptorWithOrigin,
pytest: PyTest,
test_options: TestOptions,
python_setup: PythonSetup,
) -> TestTargetSetup:
# TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage
# configuration via #7490.
adaptor = adaptor_with_origin.adaptor
test_addresses = Addresses((adaptor.address,))
# TODO(John Sirois): PexInterpreterConstraints are gathered in the same way by the
# `create_pex_from_target_closure` rule, factor up.
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](Addresses, test_addresses)
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=all_target_adaptors, python_setup=python_setup
)
# Ensure all pexes we merge via PEX_PATH to form the test runner use the interpreter constraints
# of the tests. This is handled by CreatePexFromTargetClosure, but we must pass this through for
# CreatePex requests.
create_pex = functools.partial(CreatePex, interpreter_constraints=interpreter_constraints)
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args_for_pytest = ("--not-zip-safe",)
run_coverage = test_options.values.run_coverage
plugin_file_digest: Optional[Digest] = (
await Get[Digest](InputFilesContent, get_coverage_plugin_input()) if run_coverage else None
)
pytest_pex = await Get[Pex](
CreatePex,
create_pex(
output_filename="pytest.pex",
requirements=PexRequirements(pytest.get_requirement_strings()),
additional_args=additional_args_for_pytest,
input_files_digest=plugin_file_digest,
),
)
requirements_pex = await Get[Pex](
CreatePexFromTargetClosure(
addresses=test_addresses,
output_filename="requirements.pex",
include_source_files=False,
additional_args=additional_args_for_pytest,
)
)
test_runner_pex = await Get[Pex](
CreatePex,
create_pex(
output_filename="test_runner.pex",
entry_point="pytest:main",
interpreter_constraints=interpreter_constraints,
additional_args=(
"--pex-path",
":".join(
pex_request.output_filename
# TODO(John Sirois): Support shading python binaries:
# https://github.com/pantsbuild/pants/issues/9206
# Right now any pytest transitive requirements will shadow corresponding user
# requirements which will lead to problems when APIs that are used by either
# `pytest:main` or the tests themselves break between the two versions.
for pex_request in (pytest_pex, requirements_pex)
),
),
),
)
chrooted_sources = await Get[ChrootedPythonSources](HydratedTargets(all_targets))
directories_to_merge = [
chrooted_sources.snapshot.directory_digest,
requirements_pex.directory_digest,
pytest_pex.directory_digest,
test_runner_pex.directory_digest,
]
# Get the file names for the test_target so that we can specify to Pytest precisely which files
# to test, rather than using auto-discovery.
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest([adaptor_with_origin], strip_source_roots=True)
)
specified_source_file_names = specified_source_files.snapshot.files
coverage_args = []
if run_coverage:
coveragerc = await Get[Coveragerc](
CoveragercRequest(HydratedTargets(all_targets), test_time=True)
)
directories_to_merge.append(coveragerc.digest)
packages_to_cover = get_packages_to_cover(
target=adaptor, specified_source_files=specified_source_files,
)
coverage_args = [
"--cov-report=", # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html
]
for package in packages_to_cover:
coverage_args.extend(["--cov", package])
merged_input_files = await Get[Digest](
DirectoriesToMerge(directories=tuple(directories_to_merge))
)
timeout_seconds = calculate_timeout_seconds(
timeouts_enabled=pytest.options.timeouts,
target_timeout=getattr(adaptor, "timeout", None),
timeout_default=pytest.options.timeout_default,
timeout_maximum=pytest.options.timeout_maximum,
)
return TestTargetSetup(
test_runner_pex=test_runner_pex,
args=(*pytest.options.args, *coverage_args, *sorted(specified_source_file_names)),
input_files_digest=merged_input_files,
timeout_seconds=timeout_seconds,
)
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot:
owned_deps = await Get[OwnedDependencies](DependencyOwner(
request.exported_target))
targets = HydratedTargets(od.hydrated_target for od in owned_deps)
sources = await Get[SetupPySources](SetupPySourcesRequest(targets,
py2=request.py2))
requirements = await Get[ExportedTargetRequirements](DependencyOwner(
request.exported_target))
# Nest the sources under the src/ prefix.
src_digest = await Get[Digest](DirectoryWithPrefixToAdd(
sources.digest, CHROOT_SOURCE_ROOT))
# Generate the kwargs to the setup() call.
setup_kwargs = request.exported_target.hydrated_target.adaptor.provides.setup_py_keywords.copy(
)
setup_kwargs.update({
"package_dir": {
"": CHROOT_SOURCE_ROOT
},
"packages": sources.packages,
"namespace_packages": sources.namespace_packages,
"package_data": dict(sources.package_data),
"install_requires": requirements.requirement_strs,
})
adaptor = request.exported_target.hydrated_target.adaptor
key_to_binary_spec = getattr(adaptor.provides, "binaries", {})
keys = list(key_to_binary_spec.keys())
binaries = await MultiGet(Get[HydratedTarget](
Address,
Address.parse(key_to_binary_spec[key],
relative_to=adaptor.address.spec_path)) for key in keys)
for key, binary in zip(keys, binaries):
if (not isinstance(binary.adaptor, PythonBinaryAdaptor)
or getattr(binary.adaptor, "entry_point", None) is None):
raise InvalidEntryPoint(
f"The binary {key} exported by {adaptor.address.reference()} is not a valid entry point."
)
entry_points = setup_kwargs["entry_points"] = setup_kwargs.get(
"entry_points", {})
console_scripts = entry_points["console_scripts"] = entry_points.get(
"console_scripts", [])
console_scripts.append(f"{key}={binary.adaptor.entry_point}")
# Generate the setup script.
setup_py_content = SETUP_BOILERPLATE.format(
target_address_spec=adaptor.address.reference(),
setup_kwargs_str=distutils_repr(setup_kwargs),
).encode()
extra_files_digest = await Get[Digest](
InputFilesContent([
FileContent("setup.py", setup_py_content),
FileContent(
"MANIFEST.in",
"include *.py".encode()), # Make sure setup.py is included.
]))
chroot_digest = await Get[Digest](DirectoriesToMerge(
(src_digest, extra_files_digest)))
return SetupPyChroot(chroot_digest, json.dumps(setup_kwargs,
sort_keys=True))
async def setup_pytest_for_target(
test_target: PythonTestsAdaptor,
pytest: PyTest,
test_options: TestOptions,
) -> TestTargetSetup:
# TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage
# configuration via #7490.
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
resolved_requirements_pex = await Get[Pex](
CreatePexFromTargetClosure(
build_file_addresses=BuildFileAddresses((test_target.address, )),
output_filename='pytest-with-requirements.pex',
entry_point="pytest:main",
additional_requirements=pytest.get_requirement_strings(),
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args=("--not-zip-safe", ),
include_source_files=False,
))
chrooted_sources = await Get[ChrootedPythonSources](
HydratedTargets(all_targets))
directories_to_merge = [
chrooted_sources.digest,
resolved_requirements_pex.directory_digest,
]
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = await Get[
SourceRootStrippedSources](Address, test_target.address.to_address())
coverage_args = []
test_target_sources_file_names = source_root_stripped_test_target_sources.snapshot.files
if test_options.values.run_coverage:
coveragerc_digest = await Get[Digest](
InputFilesContent, get_coveragerc_input(DEFAULT_COVERAGE_CONFIG))
directories_to_merge.append(coveragerc_digest)
packages_to_cover = get_packages_to_cover(
test_target,
source_root_stripped_file_paths=test_target_sources_file_names,
)
coverage_args = [
'--cov-report=', # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html
]
for package in packages_to_cover:
coverage_args.extend(['--cov', package])
merged_input_files = await Get[Digest](
DirectoriesToMerge(directories=tuple(directories_to_merge)))
timeout_seconds = calculate_timeout_seconds(
timeouts_enabled=pytest.options.timeouts,
target_timeout=getattr(test_target, 'timeout', None),
timeout_default=pytest.options.timeout_default,
timeout_maximum=pytest.options.timeout_maximum,
)
return TestTargetSetup(
requirements_pex=resolved_requirements_pex,
args=(*pytest.options.args, *coverage_args,
*sorted(test_target_sources_file_names)),
input_files_digest=merged_input_files,
timeout_seconds=timeout_seconds,
)
