async def create_python_binary(
field_set: PythonBinaryFieldSet, python_binary_defaults: PythonBinaryDefaults
) -> CreatedBinary:
entry_point = field_set.entry_point.value
if entry_point is None:
# TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
# we only need the path relative to the source root.
binary_sources = await Get(HydratedSources, HydrateSourcesRequest(field_set.sources))
stripped_binary_sources = await Get(
StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ())
)
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
stripped_binary_sources.snapshot.files
)
output_filename = f"{field_set.address.target_name}.pex"
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
entry_point=entry_point,
platforms=PexPlatforms.create_from_platforms_field(field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(python_binary_defaults),
)
),
)
pex = two_step_pex.pex
return CreatedBinary(digest=pex.digest, binary_name=pex.output_filename)
def get_stripped_files_for_snapshot(
paths: List[str],
*,
args: Optional[List[str]] = None,
) -> List[str]:
input_snapshot = self.make_snapshot_of_empty_files(paths)
request = SourceFiles(input_snapshot, ())
return self.get_stripped_files(request, args=args)
def test_strip_snapshot(self) -> None:
def get_stripped_files_for_snapshot(
paths: List[str],
*,
args: Optional[List[str]] = None,
) -> List[str]:
input_snapshot = self.make_snapshot_of_empty_files(paths)
request = SourceFiles(input_snapshot, ())
return self.get_stripped_files(request, args=args)
# Normal source roots
assert get_stripped_files_for_snapshot(
["src/python/project/example.py"]) == ["project/example.py"]
assert get_stripped_files_for_snapshot(
["src/python/project/example.py"], ) == ["project/example.py"]
assert get_stripped_files_for_snapshot([
"src/java/com/project/example.java"
]) == ["com/project/example.java"]
assert get_stripped_files_for_snapshot([
"tests/python/project_test/example.py"
]) == ["project_test/example.py"]
# Unrecognized source root
unrecognized_source_root = "no-source-root/example.txt"
with pytest.raises(ExecutionError) as exc:
get_stripped_files_for_snapshot([unrecognized_source_root])
assert f"NoSourceRootError: No source root found for `{unrecognized_source_root}`." in str(
exc.value)
# Support for multiple source roots
file_names = [
"src/python/project/example.py",
"src/java/com/project/example.java"
]
assert get_stripped_files_for_snapshot(file_names) == [
"com/project/example.java",
"project/example.py",
]
# Test a source root at the repo root. We have performance optimizations for this case
# because there is nothing to strip.
source_root_config = [f"--source-root-patterns={json.dumps(['/'])}"]
assert get_stripped_files_for_snapshot(
["project/f1.py", "project/f2.py"],
args=source_root_config,
) == ["project/f1.py", "project/f2.py"]
assert get_stripped_files_for_snapshot(
["dir1/f.py", "dir2/f.py"],
args=source_root_config,
) == ["dir1/f.py", "dir2/f.py"]
# Gracefully handle an empty snapshot
assert self.get_stripped_files(SourceFiles(EMPTY_SNAPSHOT, ())) == []
async def create_python_binary_run_request(
field_set: PythonBinaryFieldSet,
python_binary_defaults: PythonBinaryDefaults) -> RunRequest:
entry_point = field_set.entry_point.value
if entry_point is None:
# TODO: This is overkill? We don't need to hydrate the sources and strip snapshots,
# we only need the path relative to the source root.
binary_sources = await Get(HydratedSources,
HydrateSourcesRequest(field_set.sources))
stripped_binary_sources = await Get(
StrippedSourceFiles, SourceFiles(binary_sources.snapshot, ()))
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
stripped_binary_sources.snapshot.files)
if entry_point is None:
raise InvalidFieldException(
"You must either specify `sources` or `entry_point` for the target "
f"{repr(field_set.address)} in order to run it, but both fields were undefined."
)
transitive_targets = await Get(TransitiveTargets,
Addresses([field_set.address]))
output_filename = f"{field_set.address.target_name}.pex"
pex_request = Get(
Pex,
PexFromTargetsRequest(
addresses=Addresses([field_set.address]),
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
python_binary_defaults),
include_source_files=False,
),
)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True),
)
pex, sources = await MultiGet(pex_request, sources_request)
merged_digest = await Get(
Digest,
MergeDigests([pex.digest, sources.source_files.snapshot.digest]))
return RunRequest(
digest=merged_digest,
binary_name=pex.output_filename,
prefix_args=("-m", entry_point),
env={"PEX_EXTRA_SYS_PATH": ":".join(sources.source_roots)},
)
async def prepare_python_sources(
request: PythonSourceFilesRequest,
union_membership: UnionMembership) -> PythonSourceFiles:
sources = await Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in request.targets),
for_sources_types=request.valid_sources_types,
enable_codegen=True,
),
)
missing_init_files = await Get(
AncestorFiles,
AncestorFilesRequest("__init__.py", sources.snapshot),
)
init_injected = await Get(
Snapshot,
MergeDigests(
(sources.snapshot.digest, missing_init_files.snapshot.digest)),
)
source_root_objs = await MultiGet(
Get(SourceRoot, SourceRootRequest, SourceRootRequest.for_target(tgt))
for tgt in request.targets
if (tgt.has_field(PythonSources) or tgt.has_field(ResourcesSources) or
tgt.get(Sources).can_generate(PythonSources, union_membership) or
tgt.get(Sources).can_generate(ResourcesSources, union_membership)))
source_root_paths = {
source_root_obj.path
for source_root_obj in source_root_objs
}
return PythonSourceFiles(
SourceFiles(init_injected, sources.unrooted_files),
tuple(sorted(source_root_paths)))
async def setup(
setup_request: SetupRequest,
black: Black,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
requirements_pex_request = Get[Pex](
PexRequest(
output_filename="black.pex",
requirements=PexRequirements(black.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
black.default_interpreter_constraints
),
entry_point=black.get_entry_point(),
)
)
config_path: Optional[str] = black.options.config
config_snapshot_request = Get[Snapshot](
PathGlobs(
globs=[config_path] if config_path else [],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--black-config`",
)
)
all_source_files_request = Get[SourceFiles](
AllSourceFilesRequest(field_set.sources for field_set in setup_request.request.field_sets)
)
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest(
(field_set.sources, field_set.origin) for field_set in setup_request.request.field_sets
)
)
requests = requirements_pex_request, config_snapshot_request, specified_source_files_request
all_source_files, requirements_pex, config_snapshot, specified_source_files = (
await MultiGet(all_source_files_request, *requests)
if setup_request.request.prior_formatter_result is None
else (SourceFiles(EMPTY_SNAPSHOT), *await MultiGet(*requests))
)
all_source_files_snapshot = (
all_source_files.snapshot
if setup_request.request.prior_formatter_result is None
else setup_request.request.prior_formatter_result
)
input_digest = await Get[Digest](
MergeDigests(
(all_source_files_snapshot.digest, requirements_pex.digest, config_snapshot.digest)
)
)
address_references = ", ".join(
sorted(field_set.address.reference() for field_set in setup_request.request.field_sets)
)
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./black.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
black=black,
check_only=setup_request.check_only,
),
input_digest=input_digest,
output_files=all_source_files_snapshot.files,
description=(
f"Run Black on {pluralize(len(setup_request.request.field_sets), 'target')}: {address_references}."
),
)
return Setup(process, original_digest=all_source_files_snapshot.digest)
async def setup(
setup_request: SetupRequest,
docformatter: Docformatter,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> Setup:
requirements_pex_request = Get[Pex](
PexRequest(
output_filename="docformatter.pex",
requirements=PexRequirements(docformatter.get_requirement_specs()),
interpreter_constraints=PexInterpreterConstraints(
docformatter.default_interpreter_constraints
),
entry_point=docformatter.get_entry_point(),
)
)
all_source_files_request = Get[SourceFiles](
AllSourceFilesRequest(field_set.sources for field_set in setup_request.request.field_sets)
)
specified_source_files_request = Get[SourceFiles](
SpecifiedSourceFilesRequest(
(field_set.sources, field_set.origin) for field_set in setup_request.request.field_sets
)
)
requests = requirements_pex_request, specified_source_files_request
all_source_files, requirements_pex, specified_source_files = (
await MultiGet(all_source_files_request, *requests)
if setup_request.request.prior_formatter_result is None
else (SourceFiles(EMPTY_SNAPSHOT), *await MultiGet(*requests))
)
all_source_files_snapshot = (
all_source_files.snapshot
if setup_request.request.prior_formatter_result is None
else setup_request.request.prior_formatter_result
)
input_digest = await Get[Digest](
MergeDigests((all_source_files_snapshot.digest, requirements_pex.digest))
)
address_references = ", ".join(
sorted(field_set.address.reference() for field_set in setup_request.request.field_sets)
)
process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./docformatter.pex",
pex_args=generate_args(
specified_source_files=specified_source_files,
docformatter=docformatter,
check_only=setup_request.check_only,
),
input_digest=input_digest,
output_files=all_source_files_snapshot.files,
description=(
f"Run Docformatter on {pluralize(len(setup_request.request.field_sets), 'target')}: "
f"{address_references}."
),
)
return Setup(process, original_digest=all_source_files_snapshot.digest)