async def legacy_prepare_python_sources(
hydrated_targets: HydratedTargets, ) -> ImportablePythonSources:
stripped_sources = await Get[SourceFiles](LegacyAllSourceFilesRequest(
(ht.adaptor for ht in hydrated_targets), strip_source_roots=True))
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(
stripped_sources.snapshot))
return ImportablePythonSources(init_injected.snapshot)
async def create_coverage_config(
coverage_config_request: CoverageConfigRequest, source_root_config: SourceRootConfig
) -> CoverageConfig:
sources = await Get[SourceFiles](
AllSourceFilesRequest(
(tgt.get(Sources) for tgt in coverage_config_request.targets), strip_source_roots=False,
)
)
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(sources.snapshot))
source_roots = source_root_config.get_source_roots()
# Generate a map from source root stripped source to its source root. eg:
# {'pants/testutil/subsystem/util.py': 'src/python'}. This is so that coverage reports
# referencing /chroot/path/pants/testutil/subsystem/util.py can be mapped back to the actual
# sources they reference when generating coverage reports.
def stripped_file_with_source_root(file_name: str) -> Tuple[str, str]:
source_root_object = source_roots.find_by_path(file_name)
source_root = source_root_object.path if source_root_object is not None else ""
stripped_path = file_name[len(source_root) + 1 :]
return stripped_path, source_root
stripped_files_to_source_roots = dict(
stripped_file_with_source_root(filename)
for filename in sorted(init_injected.snapshot.files)
)
default_config = dedent(
"""
[run]
branch = True
timid = False
relative_files = True
"""
)
config_parser = configparser.ConfigParser()
config_parser.read_file(StringIO(default_config))
config_parser.set("run", "plugins", COVERAGE_PLUGIN_MODULE_NAME)
config_parser.add_section(COVERAGE_PLUGIN_MODULE_NAME)
config_parser.set(
COVERAGE_PLUGIN_MODULE_NAME,
"source_to_target_base",
json.dumps(stripped_files_to_source_roots),
)
config_parser.set(
COVERAGE_PLUGIN_MODULE_NAME, "test_time", json.dumps(coverage_config_request.is_test_time)
)
config_io_stream = StringIO()
config_parser.write(config_io_stream)
digest = await Get[Digest](
InputFilesContent(
[FileContent(".coveragerc", content=config_io_stream.getvalue().encode())]
)
)
return CoverageConfig(digest)
async def prepare_python_sources(targets: Targets) -> ImportablePythonSources:
stripped_sources = await Get[SourceFiles](AllSourceFilesRequest(
(tgt.get(Sources) for tgt in targets),
for_sources_types=(PythonSources, ResourcesSources, FilesSources),
enable_codegen=True,
strip_source_roots=True,
))
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(
stripped_sources.snapshot))
return ImportablePythonSources(init_injected.snapshot)
async def merge_coverage_data(
data_batch: PytestCoverageDataBatch,
transitive_targets: TransitiveHydratedTargets,
python_setup: PythonSetup,
coverage_setup: CoverageSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> MergedCoverageData:
"""Takes all python test results and merges their coverage data into a single sql file."""
# We start with a bunch of test results, each of which has a coverage data file called `.coverage`
# We prefix each of these with their address so that we can write them all into a single pex.
coverage_directory_digests = await MultiGet(
Get[Digest](
DirectoryWithPrefixToAdd(
directory_digest=result.test_result.coverage_data.
digest, # type: ignore[attr-defined]
prefix=result.address.path_safe_spec,
)) for result in data_batch.addresses_and_test_results
if result.test_result is not None
and result.test_result.coverage_data is not None)
sources = await Get[SourceFiles](AllSourceFilesRequest(
(ht.adaptor for ht in transitive_targets.closure),
strip_source_roots=False))
sources_with_inits_snapshot = await Get[InitInjectedSnapshot](
InjectInitRequest(sources.snapshot))
coveragerc = await Get[Coveragerc](CoveragercRequest(HydratedTargets(
transitive_targets.closure),
test_time=True))
merged_input_files: Digest = await Get(
Digest,
DirectoriesToMerge(directories=(
*coverage_directory_digests,
sources_with_inits_snapshot.snapshot.directory_digest,
coveragerc.digest,
coverage_setup.requirements_pex.directory_digest,
)),
)
prefixes = [
f"{result.address.path_safe_spec}/.coverage"
for result in data_batch.addresses_and_test_results
]
coverage_args = ["combine", *prefixes]
request = coverage_setup.requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./{coverage_setup.requirements_pex.output_filename}",
pex_args=coverage_args,
input_files=merged_input_files,
output_files=(".coverage", ),
description=f"Merge coverage reports.",
)
result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request)
return MergedCoverageData(coverage_data=result.output_directory_digest)
async def generate_coverage_report(
transitive_targets: TransitiveHydratedTargets,
python_setup: PythonSetup,
coverage_setup: CoverageSetup,
merged_coverage_data: MergedCoverageData,
coverage_toolbase: PytestCoverage,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> CoverageReport:
"""Takes all python test results and generates a single coverage report."""
requirements_pex = coverage_setup.requirements_pex
# TODO(#4535) We need a better way to do this kind of check that covers synthetic targets and rules extensibility.
python_targets = [
target for target in transitive_targets.closure
if target.adaptor.type_alias in ("python_library", "python_tests")
]
coveragerc = await Get[Coveragerc](CoveragercRequest(
HydratedTargets(python_targets)))
sources = await Get[SourceFiles](AllSourceFilesRequest(
(ht.adaptor for ht in transitive_targets.closure),
strip_source_roots=False))
sources_with_inits_snapshot = await Get[InitInjectedSnapshot](
InjectInitRequest(sources.snapshot))
merged_input_files: Digest = await Get(
Digest,
DirectoriesToMerge(directories=(
merged_coverage_data.coverage_data,
coveragerc.digest,
requirements_pex.directory_digest,
sources_with_inits_snapshot.snapshot.directory_digest,
)),
)
report_type = coverage_toolbase.options.report
coverage_args = [report_type.report_name]
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./{coverage_setup.requirements_pex.output_filename}",
pex_args=coverage_args,
input_files=merged_input_files,
output_directories=("htmlcov", ),
output_files=("coverage.xml", ),
description=f"Generate coverage report.",
)
result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request)
if report_type == ReportType.CONSOLE:
return ConsoleCoverageReport(result.stdout.decode())
return FilesystemCoverageReport(
result.output_directory_digest,
coverage_toolbase.options.report_output_path)
async def merge_coverage_data(
data_collection: PytestCoverageDataCollection,
coverage_setup: CoverageSetup,
transitive_targets: TransitiveTargets,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> MergedCoverageData:
"""Takes all Python test results and merges their coverage data into a single SQL file."""
# We start with a bunch of test results, each of which has a coverage data file called
# `.coverage`. We prefix each of these with their address so that we can write them all into a
# single PEX.
coverage_directory_digests = await MultiGet(
Get[Digest](DirectoryWithPrefixToAdd(data.digest, prefix=data.address.path_safe_spec))
for data in data_collection
)
sources = await Get[SourceFiles](
AllSourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure), strip_source_roots=False
)
)
sources_with_inits = await Get[InitInjectedSnapshot](InjectInitRequest(sources.snapshot))
coverage_config = await Get[CoverageConfig](
CoverageConfigRequest(Targets(transitive_targets.closure), is_test_time=True)
)
merged_input_files: Digest = await Get(
Digest,
DirectoriesToMerge(
directories=(
*coverage_directory_digests,
sources_with_inits.snapshot.directory_digest,
coverage_config.digest,
coverage_setup.requirements_pex.directory_digest,
)
),
)
prefixes = [f"{data.address.path_safe_spec}/.coverage" for data in data_collection]
coverage_args = ("combine", *prefixes)
process = coverage_setup.requirements_pex.create_process(
pex_path=f"./{coverage_setup.requirements_pex.output_filename}",
pex_args=coverage_args,
input_files=merged_input_files,
output_files=(".coverage",),
description=f"Merge {len(prefixes)} Pytest coverage reports.",
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
)
result = await Get[ProcessResult](Process, process)
return MergedCoverageData(coverage_data=result.output_directory_digest)
async def prepare_chrooted_python_sources(
hydrated_targets: HydratedTargets, ) -> ChrootedPythonSources:
"""Prepares Python sources by stripping the source root and injecting missing __init__.py files.
NB: This is useful for Pytest or ./pants run, but not every Python rule will need this.
For example, autoformatters like Black do not need to understand relative imports or
execute the code, so they can safely operate on the original source files without
stripping source roots.
"""
stripped_sources = await Get[SourceFiles](AllSourceFilesRequest(
(ht.adaptor for ht in hydrated_targets), strip_source_roots=True))
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(
stripped_sources.snapshot))
return ChrootedPythonSources(init_injected.snapshot)
def assert_injected(self, *, original_files: List[str],
expected_added: List[str]) -> None:
request = InjectInitRequest(
self.make_snapshot({fp: "# python code"
for fp in original_files}))
result = self.request_single_product(InitInjectedSnapshot,
request).snapshot
assert sorted(result.files) == sorted(
[*original_files, *expected_added])
# Ensure all original `__init__.py` are preserved with their original content.
materialized_original_inits = [
fc for fc in self.request_single_product(FilesContent,
result.directory_digest)
if fc.path in original_files and fc.path.endswith("__init__.py")
]
for original_init in materialized_original_inits:
assert (
original_init.content == b"# python code"
), f"{original_init} does not have its original content preserved."
async def construct_coverage_config(
source_root_config: SourceRootConfig,
coverage_config_request: CoveragercRequest) -> Coveragerc:
sources = await Get[SourceFiles](AllSourceFilesRequest(
(ht.adaptor for ht in coverage_config_request.hydrated_targets),
strip_source_roots=False,
))
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(
sources.snapshot))
source_roots = source_root_config.get_source_roots()
# Generate a map from source root stripped source to its source root. eg:
# {'pants/testutil/subsystem/util.py': 'src/python'}
# This is so coverage reports referencing /chroot/path/pants/testutil/subsystem/util.py can be mapped
# back to the actual sources they reference when generating coverage reports.
def source_root_stripped_source_and_source_root(
file_name: str) -> Tuple[str, str]:
source_root = source_roots.find_by_path(file_name)
source_root_path = source_root.path if source_root is not None else ""
source_root_stripped_path = file_name[len(source_root_path) + 1:]
return (source_root_stripped_path, source_root_path)
source_to_target_base = dict(
source_root_stripped_source_and_source_root(filename)
for filename in sorted(init_injected.snapshot.files))
config_parser = configparser.ConfigParser()
config_parser.read_file(StringIO(DEFAULT_COVERAGE_CONFIG))
ensure_section(config_parser, "run")
config_parser.set("run", "plugins", COVERAGE_PLUGIN_MODULE_NAME)
config_parser.add_section(COVERAGE_PLUGIN_MODULE_NAME)
config_parser.set(COVERAGE_PLUGIN_MODULE_NAME, "source_to_target_base",
json.dumps(source_to_target_base))
config_parser.set(COVERAGE_PLUGIN_MODULE_NAME, "test_time",
json.dumps(coverage_config_request.test_time))
config = StringIO()
config_parser.write(config)
coveragerc_digest = await Get[Digest](InputFilesContent,
get_coveragerc_input(
config.getvalue()))
return Coveragerc(coveragerc_digest)
async def generate_coverage_report(
merged_coverage_data: MergedCoverageData,
coverage_setup: CoverageSetup,
coverage_subsystem: PytestCoverage,
transitive_targets: TransitiveTargets,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> CoverageReport:
"""Takes all Python test results and generates a single coverage report."""
requirements_pex = coverage_setup.requirements_pex
python_targets = [tgt for tgt in transitive_targets.closure if tgt.has_field(PythonSources)]
coverage_config = await Get[CoverageConfig](
CoverageConfigRequest(Targets(python_targets), is_test_time=False)
)
sources = await Get[SourceFiles](
AllSourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure), strip_source_roots=False
)
)
sources_with_inits_snapshot = await Get[InitInjectedSnapshot](
InjectInitRequest(sources.snapshot)
)
merged_input_files: Digest = await Get(
Digest,
DirectoriesToMerge(
directories=(
merged_coverage_data.coverage_data,
coverage_config.digest,
requirements_pex.directory_digest,
sources_with_inits_snapshot.snapshot.directory_digest,
)
),
)
report_type = coverage_subsystem.options.report
coverage_args = [report_type.report_name]
process = requirements_pex.create_process(
pex_path=f"./{coverage_setup.requirements_pex.output_filename}",
pex_args=coverage_args,
input_files=merged_input_files,
output_directories=("htmlcov",),
output_files=("coverage.xml",),
description=f"Generate Pytest coverage report.",
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
)
result = await Get[ProcessResult](Process, process)
if report_type == ReportType.CONSOLE:
return ConsoleCoverageReport(result.stdout.decode())
report_dir = PurePath(coverage_subsystem.options.report_output_path)
report_file: Optional[PurePath] = None
if coverage_subsystem.options.report == ReportType.HTML:
report_file = report_dir / "htmlcov" / "index.html"
elif coverage_subsystem.options.report == ReportType.XML:
report_file = report_dir / "coverage.xml"
return FilesystemCoverageReport(
result_digest=result.output_directory_digest,
directory_to_materialize_to=report_dir,
report_file=report_file,
)
async def generate_coverage_report(
merged_coverage_data: MergedCoverageData,
coverage_setup: CoverageSetup,
coverage_subsystem: PytestCoverage,
transitive_targets: TransitiveTargets,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> CoverageReports:
"""Takes all Python test results and generates a single coverage report."""
requirements_pex = coverage_setup.requirements_pex
coverage_config = await Get[CoverageConfig](CoverageConfigRequest(
Targets(transitive_targets.closure), is_test_time=False))
sources = await Get[SourceFiles](AllSourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
strip_source_roots=False))
sources_with_inits_snapshot = await Get[InitInjectedSnapshot](
InjectInitRequest(sources.snapshot))
input_digest: Digest = await Get(
Digest,
MergeDigests((
merged_coverage_data.coverage_data,
coverage_config.digest,
requirements_pex.digest,
sources_with_inits_snapshot.snapshot.digest,
)),
)
report_type = coverage_subsystem.options.report
# The -i flag causes coverage to ignore files it doesn't have sources for.
# Specifically, it will ignore the injected __init__.py files, which is what we want since
# those are empty and don't correspond to a real source file the user is aware of.
coverage_args = [report_type.report_name, "-i"]
process = requirements_pex.create_process(
pex_path=f"./{coverage_setup.requirements_pex.output_filename}",
pex_args=coverage_args,
input_digest=input_digest,
output_directories=("htmlcov", ),
output_files=("coverage.xml", ),
description="Generate Pytest coverage report.",
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
)
result = await Get[ProcessResult](Process, process)
if report_type == CoverageReportType.CONSOLE:
return CoverageReports(
reports=(ConsoleCoverageReport(result.stdout.decode()), ))
report_dir = PurePath(coverage_subsystem.options.report_output_path)
report_file: Optional[PurePath] = None
if report_type == CoverageReportType.HTML:
report_file = report_dir / "htmlcov" / "index.html"
elif report_type == CoverageReportType.XML:
report_file = report_dir / "coverage.xml"
fs_report = FilesystemCoverageReport(
result_digest=result.output_digest,
directory_to_materialize_to=report_dir,
report_file=report_file,
)
return CoverageReports(reports=(fs_report, ))
