async def setup_black(wrapped_target: FormattablePythonTarget, black: Black) -> BlackSetup:
config_path: Optional[str] = black.get_options().config
config_snapshot = await Get(Snapshot, PathGlobs(include=(config_path,)))
resolved_requirements_pex = await Get(
Pex, CreatePex(
output_filename="black.pex",
requirements=PexRequirements(requirements=tuple(black.get_requirement_specs())),
interpreter_constraints=PexInterpreterConstraints(
constraint_set=tuple(black.default_interpreter_constraints)
),
entry_point=black.get_entry_point(),
)
)
sources_digest = wrapped_target.target.sources.snapshot.directory_digest
merged_input_files = await Get(
Digest,
DirectoriesToMerge(
directories=(
sources_digest,
resolved_requirements_pex.directory_digest,
config_snapshot.directory_digest,
)
),
)
return BlackSetup(config_path, resolved_requirements_pex, merged_input_files)
async def create_pex_from_target_closure(request: CreatePexFromTargetClosure,
python_setup: PythonSetup) -> Pex:
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
BuildFileAddresses, request.build_file_addresses)
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=tuple(all_target_adaptors), python_setup=python_setup)
if request.include_source_files:
chrooted_sources = await Get[ChrootedPythonSources](
HydratedTargets(all_targets))
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=request.additional_requirements)
create_pex_request = CreatePex(
output_filename=request.output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=request.entry_point,
input_files_digest=chrooted_sources.digest
if request.include_source_files else None,
additional_args=request.additional_args,
)
pex = await Get[Pex](CreatePex, create_pex_request)
return pex
async def create_pex_from_target_closure(request: CreatePexFromTargetClosure,
python_setup: PythonSetup) -> Pex:
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
Addresses, request.addresses)
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=all_target_adaptors, python_setup=python_setup)
input_digests = []
if request.additional_input_files:
input_digests.append(request.additional_input_files)
if request.include_source_files:
chrooted_sources = await Get[ChrootedPythonSources](
HydratedTargets(all_targets))
input_digests.append(chrooted_sources.snapshot.directory_digest)
merged_input_digest = await Get[Digest](
DirectoriesToMerge(directories=tuple(input_digests)))
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=request.additional_requirements)
create_pex_request = CreatePex(
output_filename=request.output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=request.entry_point,
input_files_digest=merged_input_digest,
additional_args=request.additional_args,
)
pex = await Get[Pex](CreatePex, create_pex_request)
return pex
def create_pex_and_get_all_data(
self,
*,
requirements=PexRequirements(),
entry_point=None,
interpreter_constraints=PexInterpreterConstraints(),
input_files: Optional[Digest] = None,
additional_pants_args: Tuple[str, ...] = (),
additional_pex_args: Tuple[str, ...] = (),
) -> Dict:
request = CreatePex(
output_filename="test.pex",
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=entry_point,
input_files_digest=input_files,
additional_args=additional_pex_args,
)
requirements_pex = self.request_single_product(
Pex,
Params(
request,
create_options_bootstrapper(
args=["--backend-packages2=pants.backend.python", *additional_pants_args]
),
),
)
self.scheduler.materialize_directory(
DirectoryToMaterialize(requirements_pex.directory_digest),
)
with zipfile.ZipFile(os.path.join(self.build_root, "test.pex"), "r") as pex:
with pex.open("PEX-INFO", "r") as pex_info:
pex_info_content = pex_info.readline().decode()
pex_list = pex.namelist()
return {"pex": requirements_pex, "info": json.loads(pex_info_content), "files": pex_list}
def create_pex_and_get_all_data(
self,
*,
requirements=PexRequirements(),
entry_point=None,
interpreter_constraints=PexInterpreterConstraints(),
input_files: Digest = None) -> (Dict, List[str]):
def hashify_optional_collection(iterable):
return tuple(sorted(iterable)) if iterable is not None else tuple()
request = CreatePex(
output_filename="test.pex",
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=entry_point,
input_files_digest=input_files,
)
requirements_pex = self.request_single_product(
Pex,
Params(request, PythonSetup.global_instance(),
SubprocessEnvironment.global_instance(),
PythonNativeCode.global_instance()))
self.scheduler.materialize_directory(
DirectoryToMaterialize(requirements_pex.directory_digest), )
with zipfile.ZipFile(os.path.join(self.build_root, "test.pex"),
"r") as pex:
with pex.open("PEX-INFO", "r") as pex_info:
pex_info_content = pex_info.readline().decode()
pex_list = pex.namelist()
return {
'pex': requirements_pex,
'info': json.loads(pex_info_content),
'files': pex_list
}
def get_black_input(
wrapped_target: FormattablePythonTarget,
black: Black,
) -> BlackInput:
config_path = black.get_options().config
config_snapshot = yield Get(Snapshot, PathGlobs(include=(config_path, )))
resolved_requirements_pex = yield Get(
Pex,
CreatePex(
output_filename="black.pex",
requirements=PexRequirements(
requirements=tuple(black.get_requirement_specs())),
interpreter_constraints=PexInterpreterContraints(
constraint_set=frozenset(
black.default_interpreter_constraints)),
entry_point=black.get_entry_point(),
))
target = wrapped_target.target
sources_digest = target.sources.snapshot.directory_digest
all_input_digests = [
sources_digest,
resolved_requirements_pex.directory_digest,
config_snapshot.directory_digest,
]
merged_input_files = yield Get(
Digest,
DirectoriesToMerge(directories=tuple(all_input_digests)),
)
yield BlackInput(config_path, resolved_requirements_pex,
merged_input_files)
async def setup(
request: SetupRequest,
isort: Isort,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
adaptors_with_origins = request.formatter.adaptors_with_origins
requirements_pex = await Get[Pex](CreatePex(
output_filename="isort.pex",
requirements=PexRequirements(isort.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
isort.default_interpreter_constraints),
entry_point=isort.get_entry_point(),
))
config_path: Optional[List[str]] = isort.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=config_path or (),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin="the option `--isort-config`",
))
if request.formatter.prior_formatter_result is None:
all_source_files = await Get[SourceFiles](AllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
all_source_files_snapshot = all_source_files.snapshot
else:
all_source_files_snapshot = request.formatter.prior_formatter_result
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files_snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
process_request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./isort.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
isort=isort,
check_only=request.check_only,
),
input_files=merged_input_files,
output_files=all_source_files_snapshot.files,
description=f"Run isort for {address_references}",
)
return Setup(process_request)
def create_python_binary(python_binary_adaptor: PythonBinaryAdaptor,
python_setup: PythonSetup) -> CreatedBinary:
transitive_hydrated_targets = yield Get(
TransitiveHydratedTargets,
BuildFileAddresses((python_binary_adaptor.address, )))
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=tuple(all_targets), python_setup=python_setup)
source_root_stripped_sources = yield [
Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
for target_adaptor in all_targets
]
#TODO(#8420) This way of calculating the entry point works but is a bit hackish.
entry_point = None
if hasattr(python_binary_adaptor, 'entry_point'):
entry_point = python_binary_adaptor.entry_point
else:
sources_snapshot = python_binary_adaptor.sources.snapshot
if len(sources_snapshot.files) == 1:
target = transitive_hydrated_targets.roots[0]
output = yield Get(SourceRootStrippedSources, HydratedTarget,
target)
root_filename = output.snapshot.files[0]
entry_point = PythonBinary.translate_source_path_to_py_module_specifier(
root_filename)
stripped_sources_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources
]
sources_digest = yield Get(
Digest,
DirectoriesToMerge(directories=tuple(stripped_sources_digests)))
inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
all_input_digests = [sources_digest, inits_digest.directory_digest]
merged_input_files = yield Get(
Digest, DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)))
requirements = PexRequirements.create_from_adaptors(all_target_adaptors)
output_filename = f"{python_binary_adaptor.address.target_name}.pex"
create_requirements_pex = CreatePex(
output_filename=output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=entry_point,
input_files_digest=merged_input_files,
)
pex = yield Get(Pex, CreatePex, create_requirements_pex)
yield CreatedBinary(digest=pex.directory_digest,
binary_name=pex.output_filename)
async def setup_lambdex(lambdex: Lambdex) -> LambdexSetup:
requirements_pex = await Get[Pex](CreatePex(
output_filename="lambdex.pex",
requirements=PexRequirements(lambdex.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
lambdex.default_interpreter_constraints),
entry_point=lambdex.get_entry_point(),
))
return LambdexSetup(requirements_pex=requirements_pex, )
async def setup_setuptools(setuptools: Setuptools) -> SetuptoolsSetup:
# Note that this pex has no entrypoint. We use it to run our generated setup.py, which
# in turn imports from and invokes setuptools.
requirements_pex = await Get[Pex](CreatePex(
output_filename="setuptools.pex",
requirements=PexRequirements(
requirements=tuple(setuptools.get_requirement_specs())),
interpreter_constraints=PexInterpreterConstraints(
constraint_set=tuple(setuptools.default_interpreter_constraints))))
return SetuptoolsSetup(requirements_pex=requirements_pex, )
async def setup_coverage(coverage: PytestCoverage) -> CoverageSetup:
plugin_file_digest = await Get[Digest](InputFilesContent,
get_coverage_plugin_input())
output_pex_filename = "coverage.pex"
requirements_pex = await Get[Pex](CreatePex(
output_filename=output_pex_filename,
requirements=PexRequirements(coverage.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
coverage.default_interpreter_constraints),
entry_point=coverage.get_entry_point(),
input_files_digest=plugin_file_digest,
))
return CoverageSetup(requirements_pex)
async def lint(
wrapped_target: Flake8Target,
flake8: Flake8,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if flake8.options.skip:
return LintResult.noop()
target = wrapped_target.target
# NB: Flake8 output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://flake8.pycqa.org/en/latest/user/invocation.html.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=[target] if isinstance(target, PythonTargetAdaptor) else [],
python_setup=python_setup
)
config_path: Optional[str] = flake8.options.config
config_snapshot = await Get[Snapshot](
PathGlobs(include=tuple([config_path] if config_path else []))
)
requirements_pex = await Get[Pex](
CreatePex(
output_filename="flake8.pex",
requirements=PexRequirements(requirements=tuple(flake8.get_requirement_specs())),
interpreter_constraints=interpreter_constraints,
entry_point=flake8.get_entry_point(),
)
)
merged_input_files = await Get[Digest](
DirectoriesToMerge(
directories=(
target.sources.snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)
),
)
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f'./flake8.pex',
pex_args=generate_args(wrapped_target, flake8),
input_files=merged_input_files,
description=f'Run Flake8 for {target.address.reference()}',
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest, request)
return LintResult.from_fallible_execute_process_result(result)
async def setup(
request: SetupRequest,
docformatter: Docformatter,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
adaptors_with_origins = request.formatter.adaptors_with_origins
requirements_pex = await Get[Pex](CreatePex(
output_filename="docformatter.pex",
requirements=PexRequirements(docformatter.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
docformatter.default_interpreter_constraints),
entry_point=docformatter.get_entry_point(),
))
if request.formatter.prior_formatter_result is None:
all_source_files = await Get[SourceFiles](AllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
all_source_files_snapshot = all_source_files.snapshot
else:
all_source_files_snapshot = request.formatter.prior_formatter_result
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files_snapshot.directory_digest,
requirements_pex.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
process_request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./docformatter.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
docformatter=docformatter,
check_only=request.check_only,
),
input_files=merged_input_files,
output_files=all_source_files_snapshot.files,
description=f"Run docformatter for {address_references}",
)
return Setup(process_request)
async def lint(
wrapped_target: BanditTarget,
bandit: Bandit,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if bandit.options.skip:
return LintResult.noop()
target = wrapped_target.target
# NB: Bandit output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# https://github.com/PyCQA/bandit#under-which-version-of-python-should-i-install-bandit.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=[target] if isinstance(target, PythonTargetAdaptor) else [],
python_setup=python_setup)
config_path: Optional[str] = bandit.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--bandit-config`",
))
requirements_pex = await Get[Pex](CreatePex(
output_filename="bandit.pex",
requirements=PexRequirements(
requirements=tuple(bandit.get_requirement_specs())),
interpreter_constraints=interpreter_constraints,
entry_point=bandit.get_entry_point(),
))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
target.sources.snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f'./bandit.pex',
pex_args=generate_args(wrapped_target, bandit),
input_files=merged_input_files,
description=f'Run Bandit for {target.address.reference()}',
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return LintResult.from_fallible_execute_process_result(result)
async def setup_black(black: Black) -> BlackSetup:
config_path: Optional[str] = black.get_options().config
config_snapshot = await Get[Snapshot](PathGlobs(include=(config_path, )))
requirements_pex = await Get[Pex](CreatePex(
output_filename="black.pex",
requirements=PexRequirements(
requirements=tuple(black.get_requirement_specs())),
interpreter_constraints=PexInterpreterConstraints(
constraint_set=tuple(black.default_interpreter_constraints)),
entry_point=black.get_entry_point(),
))
return BlackSetup(
requirements_pex=requirements_pex,
config_snapshot=config_snapshot,
passthrough_args=black.get_args(),
)
async def setup_isort(isort: Isort) -> IsortSetup:
config_path: Optional[List[str]] = isort.get_options().config
config_snapshot = await Get[Snapshot](PathGlobs(include=config_path or ()))
requirements_pex = await Get[Pex](CreatePex(
output_filename="isort.pex",
requirements=PexRequirements(
requirements=tuple(isort.get_requirement_specs())),
interpreter_constraints=PexInterpreterConstraints(
constraint_set=tuple(isort.default_interpreter_constraints)),
entry_point=isort.get_entry_point(),
))
return IsortSetup(
requirements_pex=requirements_pex,
config_snapshot=config_snapshot,
passthrough_args=isort.get_args(),
)
async def create_pex_from_target_closure(request: CreatePexFromTargetClosure,
python_setup: PythonSetup) -> Pex:
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
BuildFileAddresses, request.build_file_addresses)
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=tuple(all_targets), python_setup=python_setup)
merged_input_files: Optional[Digest] = None
if request.include_source_files:
source_root_stripped_sources = await MultiGet(
Get[SourceRootStrippedSources](HydratedTarget, target_adaptor)
for target_adaptor in all_targets)
stripped_sources_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources
]
sources_digest = await Get[Digest](
DirectoriesToMerge(directories=tuple(stripped_sources_digests)))
inits_digest = await Get[InjectedInitDigest](Digest, sources_digest)
all_input_digests = [sources_digest, inits_digest.directory_digest]
merged_input_files = await Get[Digest](
DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)))
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=request.additional_requirements)
create_pex_request = CreatePex(
output_filename=request.output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=request.entry_point,
input_files_digest=merged_input_files,
)
pex = await Get[Pex](CreatePex, create_pex_request)
return pex
async def setup_black(black: Black) -> BlackSetup:
config_path: Optional[str] = black.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--black-config`",
))
requirements_pex = await Get[Pex](CreatePex(
output_filename="black.pex",
requirements=PexRequirements(
requirements=tuple(black.get_requirement_specs())),
interpreter_constraints=PexInterpreterConstraints(
constraint_set=tuple(black.default_interpreter_constraints)),
entry_point=black.get_entry_point(),
))
return BlackSetup(
requirements_pex=requirements_pex,
config_snapshot=config_snapshot,
passthrough_args=tuple(black.options.args),
skip=black.options.skip,
)
async def setup_isort(isort: Isort) -> IsortSetup:
config_path: Optional[List[str]] = isort.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=config_path or (),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
conjunction=GlobExpansionConjunction.all_match,
description_of_origin="the option `--isort-config`",
))
requirements_pex = await Get[Pex](CreatePex(
output_filename="isort.pex",
requirements=PexRequirements(
requirements=tuple(isort.get_requirement_specs())),
interpreter_constraints=PexInterpreterConstraints(
constraint_set=tuple(isort.default_interpreter_constraints)),
entry_point=isort.get_entry_point(),
))
return IsortSetup(
requirements_pex=requirements_pex,
config_snapshot=config_snapshot,
passthrough_args=tuple(isort.options.args),
skip=isort.options.skip,
)
def run_black(
wrapped_target: FormattablePythonTarget,
black: Black,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> FmtResult:
config_path = black.get_options().config
config_snapshot = yield Get(Snapshot, PathGlobs(include=(config_path, )))
resolved_requirements_pex = yield Get(
Pex,
CreatePex(
output_filename="black.pex",
requirements=PexRequirements(
requirements=tuple(black.get_requirement_specs())),
interpreter_constraints=PexInterpreterContraints(
constraint_set=frozenset(
black.default_interpreter_constraints)),
entry_point=black.get_entry_point(),
))
target = wrapped_target.target
sources_digest = target.sources.snapshot.directory_digest
all_input_digests = [
sources_digest,
resolved_requirements_pex.directory_digest,
config_snapshot.directory_digest,
]
merged_input_files = yield Get(
Digest,
DirectoriesToMerge(directories=tuple(all_input_digests)),
)
# The exclude option from Black only works on recursive invocations,
# so call black with the directories in which the files are present
# and passing the full file names with the include option
dirs: Set[str] = set()
for filename in target.sources.snapshot.files:
dirs.add(f"{Path(filename).parent}")
pex_args = tuple(sorted(dirs))
if config_path:
pex_args += ("--config", config_path)
if target.sources.snapshot.files:
pex_args += ("--include", "|".join(
re.escape(f) for f in target.sources.snapshot.files))
request = resolved_requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./black.pex",
pex_args=pex_args,
input_files=merged_input_files,
output_files=target.sources.snapshot.files,
description=f'Run Black for {target.address.reference()}',
)
result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, request)
yield FmtResult(
digest=result.output_directory_digest,
stdout=result.stdout.decode(),
stderr=result.stderr.decode(),
)
async def lint(
linter: PylintLinter,
pylint: Pylint,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if pylint.options.skip:
return LintResult.noop()
adaptors_with_origins = linter.adaptors_with_origins
# Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it
# doesn't lint those direct dependencies nor does it care about transitive dependencies.
hydrated_targets = [
HydratedTarget(adaptor_with_origin.adaptor)
for adaptor_with_origin in adaptors_with_origins
]
dependencies = await MultiGet(
Get[HydratedTarget](Address, dependency)
for dependency in itertools.chain.from_iterable(
ht.adaptor.dependencies for ht in hydrated_targets))
chrooted_python_sources = await Get[ChrootedPythonSources](HydratedTargets(
[*hydrated_targets, *dependencies]))
# NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
(adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins),
python_setup=python_setup,
)
requirements_pex = await Get[Pex](CreatePex(
output_filename="pylint.pex",
requirements=PexRequirements(pylint.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=pylint.get_entry_point(),
))
config_path: Optional[str] = pylint.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
requirements_pex.directory_digest,
config_snapshot.directory_digest,
chrooted_python_sources.snapshot.directory_digest,
)), )
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest(adaptors_with_origins,
strip_source_roots=True))
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./pylint.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
pylint=pylint),
input_files=merged_input_files,
description=f"Run Pylint for {address_references}",
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return LintResult.from_fallible_execute_process_result(result)
async def lint(
linter: Flake8Linter,
flake8: Flake8,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if flake8.options.skip:
return LintResult.noop()
adaptors_with_origins = linter.adaptors_with_origins
# NB: Flake8 output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://flake8.pycqa.org/en/latest/user/invocation.html.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
(adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins),
python_setup=python_setup,
)
requirements_pex = await Get[Pex](CreatePex(
output_filename="flake8.pex",
requirements=PexRequirements(flake8.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=flake8.get_entry_point(),
))
config_path: Optional[str] = flake8.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--flake8-config`",
))
all_source_files = await Get[SourceFiles](AllSourceFilesRequest(
adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins))
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest(adaptors_with_origins))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
all_source_files.snapshot.directory_digest,
requirements_pex.directory_digest,
config_snapshot.directory_digest,
)), )
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./flake8.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
flake8=flake8),
input_files=merged_input_files,
description=f"Run Flake8 for {address_references}",
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return LintResult.from_fallible_execute_process_result(result)
async def run_python_test(
test_target: PythonTestsAdaptor, pytest: PyTest, python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment
) -> TestResult:
"""Runs pytest for one target."""
# TODO(7726): replace this with a proper API to get the `closure` for a
# TransitiveHydratedTarget.
transitive_hydrated_targets = await Get(
TransitiveHydratedTargets, BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = tuple(t.adaptor for t in all_targets)
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=tuple(all_target_adaptors), python_setup=python_setup)
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=pytest.get_requirement_strings())
resolved_requirements_pex = await Get(
Pex,
CreatePex(
output_filename=output_pytest_requirements_pex_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point="pytest:main",
))
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = await Get(
SourceRootStrippedSources, Address, test_target.address.to_address())
source_root_stripped_sources = await MultiGet(
Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
for target_adaptor in all_targets)
stripped_sources_digests = tuple(
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources)
sources_digest = await Get(
Digest, DirectoriesToMerge(directories=stripped_sources_digests))
inits_digest = await Get(InjectedInitDigest, Digest, sources_digest)
merged_input_files = await Get(
Digest,
DirectoriesToMerge(directories=(
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
)),
)
test_target_sources_file_names = sorted(
source_root_stripped_test_target_sources.snapshot.files)
request = resolved_requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f'./{output_pytest_requirements_pex_filename}',
pex_args=(*pytest.get_args(), *test_target_sources_file_names),
input_files=merged_input_files,
description=f'Run Pytest for {test_target.address.reference()}',
# TODO(#8584): hook this up to TestRunnerTaskMixin so that we can configure the default timeout
# and also use the specified max timeout time.
timeout_seconds=getattr(test_target, 'timeout', 60))
result = await Get(FallibleExecuteProcessResult, ExecuteProcessRequest,
request)
return TestResult.from_fallible_execute_process_result(result)
def run_python_test(
test_target: PythonTestsAdaptor, pytest: PyTest, python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment
) -> TestResult:
"""Runs pytest for one target."""
# TODO(7726): replace this with a proper API to get the `closure` for a
# TransitiveHydratedTarget.
transitive_hydrated_targets = yield Get(
TransitiveHydratedTargets, BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = tuple(t.adaptor for t in all_targets)
interpreter_constraints = PexInterpreterContraints.create_from_adaptors(
adaptors=tuple(all_target_adaptors), python_setup=python_setup)
# Produce a pex containing pytest and all transitive 3rdparty requirements.
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=pytest.get_requirement_strings())
resolved_requirements_pex = yield Get(
Pex,
CreatePex(
output_filename=output_pytest_requirements_pex_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point="pytest:main",
))
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = yield Get(
SourceRootStrippedSources, Address, test_target.address.to_address())
source_root_stripped_sources = yield [
Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
for target_adaptor in all_targets
]
stripped_sources_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources
]
sources_digest = yield Get(
Digest,
DirectoriesToMerge(directories=tuple(stripped_sources_digests)),
)
inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
all_input_digests = [
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
]
merged_input_files = yield Get(
Digest,
DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)),
)
test_target_sources_file_names = sorted(
source_root_stripped_test_target_sources.snapshot.files)
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
request = resolved_requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f'./{output_pytest_requirements_pex_filename}',
pex_args=test_target_sources_file_names,
input_files=merged_input_files,
description=f'Run Pytest for {test_target.address.reference()}',
)
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest,
request)
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield TestResult(
status=status,
stdout=result.stdout.decode(),
stderr=result.stderr.decode(),
)