async def legacy_prepare_python_sources(
hydrated_targets: HydratedTargets, ) -> ImportablePythonSources:
stripped_sources = await Get[SourceFiles](LegacyAllSourceFilesRequest(
(ht.adaptor for ht in hydrated_targets), strip_source_roots=True))
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(
stripped_sources.snapshot))
return ImportablePythonSources(init_injected.snapshot)
async def setup(
request: SetupRequest,
isort: Isort,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
adaptors_with_origins = request.formatter.adaptors_with_origins
requirements_pex = await Get[Pex](PexRequest(
output_filename="isort.pex",
requirements=PexRequirements(isort.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
isort.default_interpreter_constraints),
entry_point=isort.get_entry_point(),
))
config_path: Optional[List[str]] = isort.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=config_path or (),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin="the option `--isort-config`",
))
if request.formatter.prior_formatter_result is None:
all_source_files = await Get[SourceFiles](LegacyAllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
all_source_files_snapshot = all_source_files.snapshot
else:
all_source_files_snapshot = request.formatter.prior_formatter_result
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files_snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
process_request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./isort.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
isort=isort,
check_only=request.check_only,
),
input_files=merged_input_files,
output_files=all_source_files_snapshot.files,
description=f"Run isort for {address_references}",
)
return Setup(process_request)
async def merge_coverage_data(
data_batch: PytestCoverageDataBatch,
transitive_targets: TransitiveHydratedTargets,
python_setup: PythonSetup,
coverage_setup: CoverageSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> MergedCoverageData:
"""Takes all python test results and merges their coverage data into a single sql file."""
# We start with a bunch of test results, each of which has a coverage data file called `.coverage`
# We prefix each of these with their address so that we can write them all into a single pex.
coverage_directory_digests = await MultiGet(
Get[Digest](
DirectoryWithPrefixToAdd(
directory_digest=result.test_result.coverage_data.
digest, # type: ignore[attr-defined]
prefix=result.address.path_safe_spec,
)) for result in data_batch.addresses_and_test_results
if result.test_result.coverage_data is not None)
sources = await Get[SourceFiles](LegacyAllSourceFilesRequest(
(ht.adaptor for ht in transitive_targets.closure),
strip_source_roots=False))
sources_with_inits_snapshot = await Get[InitInjectedSnapshot](
InjectInitRequest(sources.snapshot))
coveragerc = await Get[Coveragerc](CoveragercRequest(HydratedTargets(
transitive_targets.closure),
test_time=True))
merged_input_files: Digest = await Get(
Digest,
DirectoriesToMerge(directories=(
*coverage_directory_digests,
sources_with_inits_snapshot.snapshot.directory_digest,
coveragerc.digest,
coverage_setup.requirements_pex.directory_digest,
)),
)
prefixes = [
f"{result.address.path_safe_spec}/.coverage"
for result in data_batch.addresses_and_test_results
]
coverage_args = ["combine", *prefixes]
request = coverage_setup.requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./{coverage_setup.requirements_pex.output_filename}",
pex_args=coverage_args,
input_files=merged_input_files,
output_files=(".coverage", ),
description=f"Merge coverage reports.",
)
result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request)
return MergedCoverageData(coverage_data=result.output_directory_digest)
async def setup(
request: SetupRequest,
docformatter: Docformatter,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
adaptors_with_origins = request.formatter.adaptors_with_origins
requirements_pex = await Get[Pex](PexRequest(
output_filename="docformatter.pex",
requirements=PexRequirements(docformatter.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
docformatter.default_interpreter_constraints),
entry_point=docformatter.get_entry_point(),
))
if request.formatter.prior_formatter_result is None:
all_source_files = await Get[SourceFiles](LegacyAllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
all_source_files_snapshot = all_source_files.snapshot
else:
all_source_files_snapshot = request.formatter.prior_formatter_result
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files_snapshot.directory_digest,
requirements_pex.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
process_request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./docformatter.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
docformatter=docformatter,
check_only=request.check_only,
),
input_files=merged_input_files,
output_files=all_source_files_snapshot.files,
description=f"Run docformatter for {address_references}",
)
return Setup(process_request)
def get_all_source_files(
self,
adaptors_with_origins: Iterable[TargetAdaptorWithOrigin],
*,
strip_source_roots: bool = False,
) -> List[str]:
request = LegacyAllSourceFilesRequest(
(adaptor_with_origin.adaptor for adaptor_with_origin in adaptors_with_origins),
strip_source_roots=strip_source_roots,
)
result = self.request_single_product(
SourceFiles, Params(request, create_options_bootstrapper())
)
return sorted(result.snapshot.files)
async def construct_coverage_config(
source_root_config: SourceRootConfig,
coverage_config_request: CoveragercRequest) -> Coveragerc:
sources = await Get[SourceFiles](LegacyAllSourceFilesRequest(
(ht.adaptor for ht in coverage_config_request.hydrated_targets),
strip_source_roots=False,
))
init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(
sources.snapshot))
source_roots = source_root_config.get_source_roots()
# Generate a map from source root stripped source to its source root. eg:
# {'pants/testutil/subsystem/util.py': 'src/python'}
# This is so coverage reports referencing /chroot/path/pants/testutil/subsystem/util.py can be mapped
# back to the actual sources they reference when generating coverage reports.
def source_root_stripped_source_and_source_root(
file_name: str) -> Tuple[str, str]:
source_root = source_roots.find_by_path(file_name)
source_root_path = source_root.path if source_root is not None else ""
source_root_stripped_path = file_name[len(source_root_path) + 1:]
return (source_root_stripped_path, source_root_path)
source_to_target_base = dict(
source_root_stripped_source_and_source_root(filename)
for filename in sorted(init_injected.snapshot.files))
config_parser = configparser.ConfigParser()
config_parser.read_file(StringIO(DEFAULT_COVERAGE_CONFIG))
ensure_section(config_parser, "run")
config_parser.set("run", "plugins", COVERAGE_PLUGIN_MODULE_NAME)
config_parser.add_section(COVERAGE_PLUGIN_MODULE_NAME)
config_parser.set(COVERAGE_PLUGIN_MODULE_NAME, "source_to_target_base",
json.dumps(source_to_target_base))
config_parser.set(COVERAGE_PLUGIN_MODULE_NAME, "test_time",
json.dumps(coverage_config_request.test_time))
config = StringIO()
config_parser.write(config)
coveragerc_digest = await Get[Digest](InputFilesContent,
get_coveragerc_input(
config.getvalue()))
return Coveragerc(coveragerc_digest)
async def lint(
linter: Flake8Linter,
flake8: Flake8,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if flake8.options.skip:
return LintResult.noop()
adaptors_with_origins = linter.adaptors_with_origins
# NB: Flake8 output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://flake8.pycqa.org/en/latest/user/invocation.html.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
(adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins),
python_setup=python_setup,
)
requirements_pex = await Get[Pex](PexRequest(
output_filename="flake8.pex",
requirements=PexRequirements(flake8.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=flake8.get_entry_point(),
))
config_path: Optional[str] = flake8.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--flake8-config`",
))
all_source_files = await Get[SourceFiles](LegacyAllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files.snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./flake8.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
flake8=flake8),
input_files=merged_input_files,
description=f"Run Flake8 for {address_references}",
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return LintResult.from_fallible_execute_process_result(result)
async def generate_coverage_report(
transitive_targets: TransitiveHydratedTargets,
python_setup: PythonSetup,
coverage_setup: CoverageSetup,
merged_coverage_data: MergedCoverageData,
coverage_toolbase: PytestCoverage,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> CoverageReport:
"""Takes all python test results and generates a single coverage report."""
requirements_pex = coverage_setup.requirements_pex
# TODO(#4535) We need a better way to do this kind of check that covers synthetic targets and rules extensibility.
python_targets = [
target for target in transitive_targets.closure
if target.adaptor.type_alias in ("python_library", "python_tests")
]
coveragerc = await Get[Coveragerc](CoveragercRequest(
HydratedTargets(python_targets)))
sources = await Get[SourceFiles](LegacyAllSourceFilesRequest(
(ht.adaptor for ht in transitive_targets.closure),
strip_source_roots=False))
sources_with_inits_snapshot = await Get[InitInjectedSnapshot](
InjectInitRequest(sources.snapshot))
merged_input_files: Digest = await Get(
Digest,
DirectoriesToMerge(directories=(
merged_coverage_data.coverage_data,
coveragerc.digest,
requirements_pex.directory_digest,
sources_with_inits_snapshot.snapshot.directory_digest,
)),
)
report_type = coverage_toolbase.options.report
coverage_args = [report_type.report_name]
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./{coverage_setup.requirements_pex.output_filename}",
pex_args=coverage_args,
input_files=merged_input_files,
output_directories=("htmlcov", ),
output_files=("coverage.xml", ),
description=f"Generate coverage report.",
)
result = await Get[ExecuteProcessResult](ExecuteProcessRequest, request)
if report_type == ReportType.CONSOLE:
return ConsoleCoverageReport(result.stdout.decode())
report_dir = PurePath(coverage_toolbase.options.report_output_path)
report_file: Optional[PurePath] = None
if coverage_toolbase.options.report == ReportType.HTML:
report_file = report_dir / "htmlcov" / "index.html"
elif coverage_toolbase.options.report == ReportType.XML:
report_file = report_dir / "coverage.xml"
return FilesystemCoverageReport(
result_digest=result.output_directory_digest,
directory_to_materialize_to=report_dir,
report_file=report_file,
)
async def bandit_lint(
linter: BanditLinter,
bandit: Bandit,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if bandit.options.skip:
return LintResult.noop()
adaptors_with_origins = linter.adaptors_with_origins
# NB: Bandit output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# https://github.com/PyCQA/bandit#under-which-version-of-python-should-i-install-bandit.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
(adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins),
python_setup=python_setup,
)
requirements_pex = await Get[Pex](PexRequest(
output_filename="bandit.pex",
requirements=PexRequirements(bandit.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=bandit.get_entry_point(),
))
config_path: Optional[str] = bandit.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--bandit-config`",
))
all_source_files = await Get[SourceFiles](LegacyAllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files.snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./bandit.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
bandit=bandit),
input_files=merged_input_files,
description=f"Run Bandit for {address_references}",
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return LintResult.from_fallible_execute_process_result(result)