async def create_pex_from_target_closure(request: CreatePexFromTargetClosure,
python_setup: PythonSetup) -> Pex:
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
BuildFileAddresses, request.build_file_addresses)
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=tuple(all_target_adaptors), python_setup=python_setup)
if request.include_source_files:
chrooted_sources = await Get[ChrootedPythonSources](
HydratedTargets(all_targets))
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=request.additional_requirements)
create_pex_request = CreatePex(
output_filename=request.output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=request.entry_point,
input_files_digest=chrooted_sources.digest
if request.include_source_files else None,
additional_args=request.additional_args,
)
pex = await Get[Pex](CreatePex, create_pex_request)
return pex
async def create_pex_from_target_closure(request: CreatePexFromTargetClosure,
python_setup: PythonSetup) -> Pex:
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
Addresses, request.addresses)
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=all_target_adaptors, python_setup=python_setup)
input_digests = []
if request.additional_input_files:
input_digests.append(request.additional_input_files)
if request.include_source_files:
chrooted_sources = await Get[ChrootedPythonSources](
HydratedTargets(all_targets))
input_digests.append(chrooted_sources.snapshot.directory_digest)
merged_input_digest = await Get[Digest](
DirectoriesToMerge(directories=tuple(input_digests)))
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=request.additional_requirements)
create_pex_request = CreatePex(
output_filename=request.output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=request.entry_point,
input_files_digest=merged_input_digest,
additional_args=request.additional_args,
)
pex = await Get[Pex](CreatePex, create_pex_request)
return pex
def create_python_binary(python_binary_adaptor: PythonBinaryAdaptor,
python_setup: PythonSetup) -> CreatedBinary:
transitive_hydrated_targets = yield Get(
TransitiveHydratedTargets,
BuildFileAddresses((python_binary_adaptor.address, )))
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=tuple(all_targets), python_setup=python_setup)
source_root_stripped_sources = yield [
Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
for target_adaptor in all_targets
]
#TODO(#8420) This way of calculating the entry point works but is a bit hackish.
entry_point = None
if hasattr(python_binary_adaptor, 'entry_point'):
entry_point = python_binary_adaptor.entry_point
else:
sources_snapshot = python_binary_adaptor.sources.snapshot
if len(sources_snapshot.files) == 1:
target = transitive_hydrated_targets.roots[0]
output = yield Get(SourceRootStrippedSources, HydratedTarget,
target)
root_filename = output.snapshot.files[0]
entry_point = PythonBinary.translate_source_path_to_py_module_specifier(
root_filename)
stripped_sources_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources
]
sources_digest = yield Get(
Digest,
DirectoriesToMerge(directories=tuple(stripped_sources_digests)))
inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
all_input_digests = [sources_digest, inits_digest.directory_digest]
merged_input_files = yield Get(
Digest, DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)))
requirements = PexRequirements.create_from_adaptors(all_target_adaptors)
output_filename = f"{python_binary_adaptor.address.target_name}.pex"
create_requirements_pex = CreatePex(
output_filename=output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=entry_point,
input_files_digest=merged_input_files,
)
pex = yield Get(Pex, CreatePex, create_requirements_pex)
yield CreatedBinary(digest=pex.directory_digest,
binary_name=pex.output_filename)
async def get_requirements(
dep_owner: DependencyOwner) -> ExportedTargetRequirements:
tht = await Get[TransitiveHydratedTargets](BuildFileAddresses(
[dep_owner.exported_target.hydrated_target.address]))
ownable_tgts = [tgt for tgt in tht.closure if is_ownable_target(tgt)]
owners = await MultiGet(Get[ExportedTarget](OwnedDependency(ht))
for ht in ownable_tgts)
owned_by_us: Set[HydratedTarget] = set()
owned_by_others: Set[HydratedTarget] = set()
for tgt, owner in zip(ownable_tgts, owners):
(owned_by_us
if owner == dep_owner.exported_target else owned_by_others).add(tgt)
# Get all 3rdparty deps of our owned deps.
#
# Note that we need only consider requirements that are direct dependencies of our owned deps:
# If T depends on R indirectly, then it must be via some direct deps U1, U2, ... For each such U,
# if U is in the owned deps then we'll pick up R through U. And if U is not in the owned deps
# then it's owned by an exported target ET, and so R will be in the requirements for ET, and we
# will require ET.
#
# TODO: Note that this logic doesn't account for indirection via dep aggregator targets, of type
# `target`. But we don't have those in v2 (yet) anyway. Plus, as we move towards buildgen and/or
# stricter build graph hygiene, it makes sense to require that targets directly declare their
# true dependencies. Plus, in the specific realm of setup-py, since we must exclude indirect
# deps across exported target boundaries, it's not a big stretch to just insist that
# requirements must be direct deps.
direct_deps_addrs = tuple(
{dep
for ht in owned_by_us for dep in ht.dependencies})
direct_deps_tgts = await MultiGet(Get[HydratedTarget](Address, a)
for a in direct_deps_addrs)
reqs = PexRequirements.create_from_adaptors(tgt.adaptor
for tgt in direct_deps_tgts)
req_strs = list(reqs.requirements)
# Add the requirements on any exported targets on which we depend.
exported_targets_we_depend_on = await MultiGet(
Get[ExportedTarget](OwnedDependency(ht)) for ht in owned_by_others)
req_strs.extend(
sorted(et.hydrated_target.adaptor.provides.requirement
for et in set(exported_targets_we_depend_on)))
return ExportedTargetRequirements(tuple(req_strs))
async def create_pex_from_target_closure(request: CreatePexFromTargetClosure,
python_setup: PythonSetup) -> Pex:
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
BuildFileAddresses, request.build_file_addresses)
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=tuple(all_targets), python_setup=python_setup)
merged_input_files: Optional[Digest] = None
if request.include_source_files:
source_root_stripped_sources = await MultiGet(
Get[SourceRootStrippedSources](HydratedTarget, target_adaptor)
for target_adaptor in all_targets)
stripped_sources_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources
]
sources_digest = await Get[Digest](
DirectoriesToMerge(directories=tuple(stripped_sources_digests)))
inits_digest = await Get[InjectedInitDigest](Digest, sources_digest)
all_input_digests = [sources_digest, inits_digest.directory_digest]
merged_input_files = await Get[Digest](
DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)))
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=request.additional_requirements)
create_pex_request = CreatePex(
output_filename=request.output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=request.entry_point,
input_files_digest=merged_input_files,
)
pex = await Get[Pex](CreatePex, create_pex_request)
return pex
async def legacy_pex_from_targets(request: LegacyPexFromTargetsRequest,
python_setup: PythonSetup) -> PexRequest:
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
Addresses, request.addresses)
all_targets = transitive_hydrated_targets.closure
python_targets = [
t for t in all_targets if isinstance(t.adaptor, PythonTargetAdaptor)
]
resource_targets = [
t for t in all_targets
if isinstance(t.adaptor, (FilesAdaptor, ResourcesAdaptor))
]
all_target_adaptors = [t.adaptor for t in all_targets]
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=all_target_adaptors, python_setup=python_setup)
input_digests = []
if request.additional_input_files:
input_digests.append(request.additional_input_files)
if request.include_source_files:
prepared_sources = await Get[ImportablePythonSources](
HydratedTargets(python_targets + resource_targets))
input_digests.append(prepared_sources.snapshot.directory_digest)
merged_input_digest = await Get[Digest](
DirectoriesToMerge(directories=tuple(input_digests)))
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=request.additional_requirements)
return PexRequest(
output_filename=request.output_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point=request.entry_point,
input_files_digest=merged_input_digest,
additional_args=request.additional_args,
)
def run_python_test(
test_target: PythonTestsAdaptor, pytest: PyTest, python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment
) -> TestResult:
"""Runs pytest for one target."""
# TODO(7726): replace this with a proper API to get the `closure` for a
# TransitiveHydratedTarget.
transitive_hydrated_targets = yield Get(
TransitiveHydratedTargets, BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = tuple(t.adaptor for t in all_targets)
interpreter_constraints = PexInterpreterContraints.create_from_adaptors(
adaptors=tuple(all_target_adaptors), python_setup=python_setup)
# Produce a pex containing pytest and all transitive 3rdparty requirements.
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=pytest.get_requirement_strings())
resolved_requirements_pex = yield Get(
Pex,
CreatePex(
output_filename=output_pytest_requirements_pex_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point="pytest:main",
))
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = yield Get(
SourceRootStrippedSources, Address, test_target.address.to_address())
source_root_stripped_sources = yield [
Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
for target_adaptor in all_targets
]
stripped_sources_digests = [
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources
]
sources_digest = yield Get(
Digest,
DirectoriesToMerge(directories=tuple(stripped_sources_digests)),
)
inits_digest = yield Get(InjectedInitDigest, Digest, sources_digest)
all_input_digests = [
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
]
merged_input_files = yield Get(
Digest,
DirectoriesToMerge,
DirectoriesToMerge(directories=tuple(all_input_digests)),
)
test_target_sources_file_names = sorted(
source_root_stripped_test_target_sources.snapshot.files)
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
request = resolved_requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f'./{output_pytest_requirements_pex_filename}',
pex_args=test_target_sources_file_names,
input_files=merged_input_files,
description=f'Run Pytest for {test_target.address.reference()}',
)
result = yield Get(FallibleExecuteProcessResult, ExecuteProcessRequest,
request)
status = Status.SUCCESS if result.exit_code == 0 else Status.FAILURE
yield TestResult(
status=status,
stdout=result.stdout.decode(),
stderr=result.stderr.decode(),
)
async def run_python_test(
test_target: PythonTestsAdaptor, pytest: PyTest, python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment
) -> TestResult:
"""Runs pytest for one target."""
# TODO(7726): replace this with a proper API to get the `closure` for a
# TransitiveHydratedTarget.
transitive_hydrated_targets = await Get(
TransitiveHydratedTargets, BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
all_target_adaptors = tuple(t.adaptor for t in all_targets)
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
adaptors=tuple(all_target_adaptors), python_setup=python_setup)
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
requirements = PexRequirements.create_from_adaptors(
adaptors=all_target_adaptors,
additional_requirements=pytest.get_requirement_strings())
resolved_requirements_pex = await Get(
Pex,
CreatePex(
output_filename=output_pytest_requirements_pex_filename,
requirements=requirements,
interpreter_constraints=interpreter_constraints,
entry_point="pytest:main",
))
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = await Get(
SourceRootStrippedSources, Address, test_target.address.to_address())
source_root_stripped_sources = await MultiGet(
Get(SourceRootStrippedSources, HydratedTarget, target_adaptor)
for target_adaptor in all_targets)
stripped_sources_digests = tuple(
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources)
sources_digest = await Get(
Digest, DirectoriesToMerge(directories=stripped_sources_digests))
inits_digest = await Get(InjectedInitDigest, Digest, sources_digest)
merged_input_files = await Get(
Digest,
DirectoriesToMerge(directories=(
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
)),
)
test_target_sources_file_names = sorted(
source_root_stripped_test_target_sources.snapshot.files)
request = resolved_requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f'./{output_pytest_requirements_pex_filename}',
pex_args=(*pytest.get_args(), *test_target_sources_file_names),
input_files=merged_input_files,
description=f'Run Pytest for {test_target.address.reference()}',
# TODO(#8584): hook this up to TestRunnerTaskMixin so that we can configure the default timeout
# and also use the specified max timeout time.
timeout_seconds=getattr(test_target, 'timeout', 60))
result = await Get(FallibleExecuteProcessResult, ExecuteProcessRequest,
request)
return TestResult.from_fallible_execute_process_result(result)
