async def resolve_pex_entry_point(request: ResolvePexEntryPointRequest) -> ResolvedPexEntryPoint:
ep_val = request.entry_point_field.value
if ep_val is None:
return ResolvedPexEntryPoint(None, file_name_used=False)
address = request.entry_point_field.address
# We support several different schemes:
# 1) `path.to.module` => preserve exactly.
# 2) `path.to.module:func` => preserve exactly.
# 3) `app.py` => convert into `path.to.app`.
# 4) `app.py:func` => convert into `path.to.app:func`.
# If it's already a module (cases #1 and #2), simply use that. Otherwise, convert the file name
# into a module path (cases #3 and #4).
if not ep_val.module.endswith(".py"):
return ResolvedPexEntryPoint(ep_val, file_name_used=False)
# Use the engine to validate that the file exists and that it resolves to only one file.
full_glob = os.path.join(address.spec_path, ep_val.module)
entry_point_paths = await Get(
Paths,
PathGlobs(
[full_glob],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=f"{address}'s `{request.entry_point_field.alias}` field",
),
)
# We will have already raised if the glob did not match, i.e. if there were no files. But
# we need to check if they used a file glob (`*` or `**`) that resolved to >1 file.
if len(entry_point_paths.files) != 1:
raise InvalidFieldException(
f"Multiple files matched for the `{request.entry_point_field.alias}` "
f"{ep_val.spec!r} for the target {address}, but only one file expected. Are you using "
f"a glob, rather than a file name?\n\n"
f"All matching files: {list(entry_point_paths.files)}."
)
entry_point_path = entry_point_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
stripped_source_path = os.path.relpath(entry_point_path, source_root.path)
module_base, _ = os.path.splitext(stripped_source_path)
normalized_path = module_base.replace(os.path.sep, ".")
return ResolvedPexEntryPoint(
dataclasses.replace(ep_val, module=normalized_path), file_name_used=True
)
async def resolve_python_aws_handler(
request: ResolvePythonAwsHandlerRequest,
) -> ResolvedPythonAwsHandler:
handler_val = request.field.value
field_alias = request.field.alias
address = request.field.address
path, _, func = handler_val.partition(":")
# If it's already a module, simply use that. Otherwise, convert the file name into a module
# path.
if not path.endswith(".py"):
return ResolvedPythonAwsHandler(handler_val, file_name_used=False)
# Use the engine to validate that the file exists and that it resolves to only one file.
full_glob = os.path.join(address.spec_path, path)
handler_paths = await Get(
Paths,
PathGlobs(
[full_glob],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=f"{address}'s `{field_alias}` field",
),
)
# We will have already raised if the glob did not match, i.e. if there were no files. But
# we need to check if they used a file glob (`*` or `**`) that resolved to >1 file.
if len(handler_paths.files) != 1:
raise InvalidFieldException(
softwrap(
f"""
Multiple files matched for the `{field_alias}` {repr(handler_val)} for the target
{address}, but only one file expected.
Are you using a glob, rather than a file name?
All matching files: {list(handler_paths.files)}.
"""
)
)
handler_path = handler_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(handler_path),
)
stripped_source_path = os.path.relpath(handler_path, source_root.path)
module_base, _ = os.path.splitext(stripped_source_path)
normalized_path = module_base.replace(os.path.sep, ".")
return ResolvedPythonAwsHandler(f"{normalized_path}:{func}", file_name_used=True)
async def package_pex_binary(
field_set: PexBinaryFieldSet,
pex_binary_defaults: PexBinaryDefaults,
global_options: GlobalOptions,
) -> BuiltPackage:
entry_point = field_set.entry_point.value
if entry_point is None:
binary_source_paths = await Get(
Paths, PathGlobs,
field_set.sources.path_globs(FilesNotFoundBehavior.error))
if len(binary_source_paths.files) != 1:
raise InvalidFieldException(
"No `entry_point` was set for the target "
f"{repr(field_set.address)}, so it must have exactly one source, but it has "
f"{len(binary_source_paths.files)}")
entry_point_path = binary_source_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
entry_point = PexBinarySources.translate_source_file_to_entry_point(
os.path.relpath(entry_point_path, source_root.path))
output_filename = field_set.output_path.value_or_default(
field_set.address,
file_ending="pex",
use_legacy_format=global_options.options.pants_distdir_legacy_paths,
)
two_step_pex = await Get(
TwoStepPex,
TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=[field_set.address],
internal_only=False,
entry_point=entry_point,
platforms=PexPlatforms.create_from_platforms_field(
field_set.platforms),
output_filename=output_filename,
additional_args=field_set.generate_additional_args(
pex_binary_defaults),
)),
)
return BuiltPackage(two_step_pex.pex.digest,
(BuiltPackageArtifact(output_filename), ))
def _find_root(
path: str,
patterns: Iterable[str],
marker_filenames: Optional[Iterable[str]] = None,
existing_marker_files: Optional[Iterable[str]] = None,
) -> Optional[str]:
source_root_config = create_subsystem(
SourceRootConfig,
root_patterns=list(patterns),
marker_filenames=list(marker_filenames or []),
)
# This inner function is passed as the callable to the mock, to allow recursion in the rule.
def _mock_fs_check(pathglobs: PathGlobs) -> Snapshot:
for glob in pathglobs.globs:
if glob in (existing_marker_files or []):
d, f = os.path.split(pathglobs.globs[0])
return Snapshot(digest=Digest("111", 111),
files=(f, ),
dirs=(d, ))
return Snapshot(digest=Digest("000", 000), files=tuple(), dirs=tuple())
def _do_find_root(src_root_req: SourceRootRequest) -> OptionalSourceRoot:
return cast(
OptionalSourceRoot,
run_rule_with_mocks(
get_optional_source_root,
rule_args=[src_root_req, source_root_config],
mock_gets=[
MockGet(
product_type=OptionalSourceRoot,
subject_type=SourceRootRequest,
mock=_do_find_root,
),
MockGet(product_type=Snapshot,
subject_type=PathGlobs,
mock=_mock_fs_check),
],
),
)
source_root = _do_find_root(SourceRootRequest(PurePath(path))).source_root
return None if source_root is None else source_root.path
async def generate_scala_from_thrift_with_scrooge(
request: GenerateScalaFromThriftRequest, ) -> GeneratedSources:
result = await Get(
GeneratedScroogeThriftSources,
GenerateScroogeThriftSourcesRequest(
thrift_source_field=request.protocol_target[ThriftSourceField],
lang_id="scala",
lang_name="Scala",
),
)
source_root = await Get(
SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target))
source_root_restored = (await Get(
Snapshot, AddPrefix(result.snapshot.digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, result.snapshot.digest))
return GeneratedSources(source_root_restored)
async def resolve_pex_entry_point(
request: ResolvePexEntryPointRequest) -> ResolvedPexEntryPoint:
if request.entry_point_field.value:
return ResolvedPexEntryPoint(request.entry_point_field.value)
binary_source_paths = await Get(
Paths, PathGlobs,
request.sources.path_globs(FilesNotFoundBehavior.error))
if len(binary_source_paths.files) != 1:
raise InvalidFieldException(
"No `entry_point` was set for the target "
f"{repr(request.sources.address)}, so it must have exactly one source, but it has "
f"{len(binary_source_paths.files)}.")
entry_point_path = binary_source_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
stripped_source_path = os.path.relpath(entry_point_path, source_root.path)
module_base, _ = os.path.splitext(stripped_source_path)
return ResolvedPexEntryPoint(module_base.replace(os.path.sep, "."))
async def get_ancestor_init_py(targets: Targets) -> AncestorInitPyFiles:
"""Find any ancestor __init__.py files for the given targets.
Includes sibling __init__.py files. Returns the files stripped of their source roots.
"""
sources = await Get[SourceFiles](AllSourceFilesRequest(
(tgt.get(Sources) for tgt in targets),
for_sources_types=(PythonSources, ),
enable_codegen=True,
))
# Find the ancestors of all dirs containing .py files, including those dirs themselves.
source_dir_ancestors: Set[Tuple[
str, str]] = set() # Items are (src_root, path incl. src_root).
source_roots = await MultiGet(
Get[SourceRoot](SourceRootRequest, SourceRootRequest.for_file(path))
for path in sources.files)
for path, source_root in zip(sources.files, source_roots):
source_dir_ancestor = os.path.dirname(path)
# Do not allow the repository root to leak (i.e., '.' should not be a package in setup.py).
while source_dir_ancestor != source_root.path:
source_dir_ancestors.add((source_root.path, source_dir_ancestor))
source_dir_ancestor = os.path.dirname(source_dir_ancestor)
source_dir_ancestors_list = list(
source_dir_ancestors) # To force a consistent order.
# Note that we must MultiGet single globs instead of a a single Get for all the globs, because
# we match each result to its originating glob (see use of zip below).
ancestor_init_py_snapshots = await MultiGet(
Get[Snapshot](
PathGlobs,
PathGlobs([os.path.join(source_dir_ancestor[1], "__init__.py")]))
for source_dir_ancestor in source_dir_ancestors_list)
source_root_stripped_ancestor_init_pys = await MultiGet(
Get[Digest](RemovePrefix(snapshot.digest, source_dir_ancestor[0]))
for snapshot, source_dir_ancestor in zip(ancestor_init_py_snapshots,
source_dir_ancestors_list))
return AncestorInitPyFiles(source_root_stripped_ancestor_init_pys)
async def all_roots(source_root_config: SourceRootConfig) -> AllSourceRoots:
source_root_pattern_matcher = source_root_config.get_pattern_matcher()
# Create globs corresponding to all source root patterns.
all_paths: Set[str] = set()
for path in source_root_pattern_matcher.get_patterns():
if path == "/":
all_paths.add("**")
elif path.startswith("/"):
all_paths.add(f"{path[1:]}/")
else:
all_paths.add(f"**/{path}/")
# Match the patterns against actual files, to find the roots that actually exist.
snapshot = await Get[Snapshot](PathGlobs(globs=sorted(all_paths)))
responses = await MultiGet(
Get[OptionalSourceRoot](SourceRootRequest(PurePath(d))) for d in snapshot.dirs
)
all_source_roots = {
response.source_root for response in responses if response.source_root is not None
}
return AllSourceRoots(all_source_roots)
async def generate_java_from_avro(
request: GenerateJavaFromAvroRequest, ) -> GeneratedSources:
sources = await Get(
HydratedSources,
HydrateSourcesRequest(request.protocol_target[AvroSourceField]))
compile_results = await MultiGet(
Get(CompiledAvroSource,
CompileAvroSourceRequest(sources.snapshot.digest, path))
for path in sources.snapshot.files)
merged_output_digest, source_root = await MultiGet(
Get(Digest, MergeDigests([r.output_digest for r in compile_results])),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(merged_output_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, merged_output_digest))
return GeneratedSources(source_root_restored)
async def generate_java_from_thrift(
request: GenerateJavaFromThriftRequest,
thrift_java: ApacheThriftJavaSubsystem,
) -> GeneratedSources:
result = await Get(
GeneratedThriftSources,
GenerateThriftSourcesRequest(
thrift_source_field=request.protocol_target[ThriftSourceField],
lang_id="java",
lang_options=thrift_java.gen_options,
lang_name="Java",
),
)
source_root = await Get(
SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target))
source_root_restored = (await Get(
Snapshot, AddPrefix(result.snapshot.digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, result.snapshot.digest))
return GeneratedSources(source_root_restored)
async def prepare_python_sources(
request: PythonSourceFilesRequest,
union_membership: UnionMembership) -> PythonSourceFiles:
sources = await Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in request.targets),
for_sources_types=request.valid_sources_types,
enable_codegen=True,
),
)
missing_init_files = await Get(
AncestorFiles,
AncestorFilesRequest("__init__.py", sources.snapshot),
)
init_injected = await Get(
Snapshot,
MergeDigests(
(sources.snapshot.digest, missing_init_files.snapshot.digest)),
)
source_root_objs = await MultiGet(
Get(SourceRoot, SourceRootRequest, SourceRootRequest.for_target(tgt))
for tgt in request.targets
if (tgt.has_field(PythonSources) or tgt.has_field(ResourcesSources) or
tgt.get(Sources).can_generate(PythonSources, union_membership) or
tgt.get(Sources).can_generate(ResourcesSources, union_membership)))
source_root_paths = {
source_root_obj.path
for source_root_obj in source_root_objs
}
return PythonSourceFiles(
SourceFiles(init_injected, sources.unrooted_files),
tuple(sorted(source_root_paths)))
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest,
protoc: Protoc,
grpc_python_plugin: GrpcPythonPlugin,
python_protobuf_subsystem: PythonProtobufSubsystem,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
# TODO(#10917): Use TransitiveTargets instead of TransitiveTargetsLite.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequestLite([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSources, ),
),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSources]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
# To run the MyPy Protobuf plugin, we first install it with Pex, then extract the wheels and
# point Protoc to the extracted wheels with its `--plugin` argument.
extracted_mypy_wheels = None
if python_protobuf_subsystem.mypy_plugin:
mypy_pex = await Get(
Pex,
PexRequest(
output_filename="mypy_protobuf.pex",
internal_only=True,
requirements=PexRequirements(
[python_protobuf_subsystem.mypy_plugin_version]),
# This is solely to ensure that we use an appropriate interpreter when resolving
# the distribution. We don't actually run the distribution directly with Python,
# as we extract out its binary.
interpreter_constraints=PexInterpreterConstraints(
["CPython>=3.5"]),
),
)
extracted_mypy_wheels = await Get(ExtractedPexDistributions, Pex,
mypy_pex)
downloaded_grpc_plugin = (await Get(
DownloadedExternalTool,
ExternalToolRequest,
grpc_python_plugin.get_request(Platform.current),
) if request.protocol_target.get(ProtobufGrcpToggle).value else None)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
if extracted_mypy_wheels:
unmerged_digests.append(extracted_mypy_wheels.digest)
if downloaded_grpc_plugin:
unmerged_digests.append(downloaded_grpc_plugin.digest)
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--python_out", output_dir]
if extracted_mypy_wheels:
mypy_plugin_path = next(
p for p in extracted_mypy_wheels.wheel_directory_paths
if p.startswith(".deps/mypy_protobuf-"))
argv.extend([
f"--plugin=protoc-gen-mypy={mypy_plugin_path}/bin/protoc-gen-mypy",
"--mypy_out",
output_dir,
])
if downloaded_grpc_plugin:
argv.extend([
f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}",
"--grpc_out", output_dir
])
argv.extend(target_sources_stripped.snapshot.files)
env = {}
if extracted_mypy_wheels:
env["PYTHONPATH"] = ":".join(
extracted_mypy_wheels.wheel_directory_paths)
result = await Get(
ProcessResult,
Process(
argv,
env=env,
input_digest=input_digest,
description=
f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_library's source root.
source_root_request = SourceRootRequest.for_target(
request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest,
protoc: Protoc,
grpc_python_plugin: GrpcPythonPlugin,
python_protobuf_subsystem: PythonProtobufSubsystem,
python_protobuf_mypy_plugin: PythonProtobufMypyPlugin,
pex_environment: PexEnvironment,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
protoc_gen_mypy_script = "protoc-gen-mypy"
protoc_gen_mypy_grpc_script = "protoc-gen-mypy_grpc"
mypy_pex = None
mypy_request = PexRequest(
output_filename="mypy_protobuf.pex",
internal_only=True,
requirements=python_protobuf_mypy_plugin.pex_requirements(),
interpreter_constraints=python_protobuf_mypy_plugin.
interpreter_constraints,
)
if python_protobuf_subsystem.mypy_plugin:
mypy_pex = await Get(
VenvPex,
VenvPexRequest(bin_names=[protoc_gen_mypy_script],
pex_request=mypy_request),
)
if request.protocol_target.get(ProtobufGrpcToggleField).value:
mypy_info = await Get(PexResolveInfo, VenvPex, mypy_pex)
# In order to generate stubs for gRPC code, we need mypy-protobuf 2.0 or above.
if any(dist_info.project_name == "mypy-protobuf"
and dist_info.version.major >= 2
for dist_info in mypy_info):
# TODO: Use `pex_path` once VenvPex stores a Pex field.
mypy_pex = await Get(
VenvPex,
VenvPexRequest(
bin_names=[
protoc_gen_mypy_script, protoc_gen_mypy_grpc_script
],
pex_request=mypy_request,
),
)
downloaded_grpc_plugin = (await Get(
DownloadedExternalTool,
ExternalToolRequest,
grpc_python_plugin.get_request(Platform.current),
) if request.protocol_target.get(ProtobufGrpcToggleField).value else None)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
if mypy_pex:
unmerged_digests.append(mypy_pex.digest)
if downloaded_grpc_plugin:
unmerged_digests.append(downloaded_grpc_plugin.digest)
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--python_out", output_dir]
if mypy_pex:
argv.extend([
f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}",
"--mypy_out",
output_dir,
])
if downloaded_grpc_plugin:
argv.extend([
f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}",
"--grpc_out", output_dir
])
if mypy_pex and protoc_gen_mypy_grpc_script in mypy_pex.bin:
argv.extend([
f"--plugin=protoc-gen-mypy_grpc={mypy_pex.bin[protoc_gen_mypy_grpc_script].argv0}",
"--mypy_grpc_out",
output_dir,
])
argv.extend(target_sources_stripped.snapshot.files)
result = await Get(
ProcessResult,
Process(
argv,
input_digest=input_digest,
description=
f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
append_only_caches=pex_environment.in_sandbox(
working_directory=None).append_only_caches,
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_source's source root.
source_root_request = SourceRootRequest.for_target(
request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest, protoc: Protoc
) -> GeneratedSources:
download_protoc_request = Get(
DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)
)
output_dir = "_generated_files"
# TODO(#9650): replace this with a proper intrinsic to create empty directories.
create_output_dir_request = Get(
ProcessResult,
Process(
("/bin/mkdir", output_dir),
description=f"Create the directory {output_dir}",
level=LogLevel.DEBUG,
output_directories=(output_dir,),
),
)
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(TransitiveTargets, Addresses([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSources,),
),
)
target_stripped_sources_request = Get(
StrippedSourceFiles, SourceFilesRequest([request.protocol_target[ProtobufSources]]),
)
(
downloaded_protoc_binary,
create_output_dir_result,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
input_digest = await Get(
Digest,
MergeDigests(
(
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
create_output_dir_result.output_digest,
)
),
)
result = await Get(
ProcessResult,
Process(
(
downloaded_protoc_binary.exe,
"--python_out",
output_dir,
*target_sources_stripped.snapshot.files,
),
input_digest=input_digest,
description=f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir,),
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_library's source root.
source_root_request = SourceRootRequest.for_target(request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (
await Get(Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "."
else await Get(Snapshot, Digest, normalized_digest)
)
return GeneratedSources(source_root_restored)
async def generate_scala_from_protobuf(
request: GenerateScalaFromProtobufRequest,
protoc: Protoc,
scalapb: ScalaPBSubsystem,
shim_classfiles: ScalaPBShimCompiledClassfiles,
jdk: InternalJdk,
) -> GeneratedSources:
output_dir = "_generated_files"
toolcp_relpath = "__toolcp"
shimcp_relpath = "__shimcp"
plugins_relpath = "__plugins"
protoc_relpath = "__protoc"
lockfile_request = await Get(GenerateJvmLockfileFromTool,
ScalapbcToolLockfileSentinel())
(
downloaded_protoc_binary,
tool_classpath,
empty_output_dir,
transitive_targets,
inherit_env,
) = await MultiGet(
Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current)),
Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)),
Get(Digest, CreateDigest([Directory(output_dir)])),
Get(TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address])),
# Need PATH so that ScalaPB can invoke `mkfifo`.
Get(Environment, EnvironmentRequest(requested=["PATH"])),
)
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_sources_stripped, target_sources_stripped = await MultiGet(
Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
),
Get(StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]
])),
)
merged_jvm_plugins_digest = EMPTY_DIGEST
maybe_jvm_plugins_setup_args: tuple[str, ...] = ()
maybe_jvm_plugins_output_args: tuple[str, ...] = ()
jvm_plugins = scalapb.jvm_plugins
if jvm_plugins:
materialized_jvm_plugins = await Get(
MaterializedJvmPlugins, MaterializeJvmPluginsRequest(jvm_plugins))
merged_jvm_plugins_digest = materialized_jvm_plugins.digest
maybe_jvm_plugins_setup_args = materialized_jvm_plugins.setup_args(
plugins_relpath)
maybe_jvm_plugins_output_args = tuple(
f"--{plugin.name}_out={output_dir}"
for plugin in materialized_jvm_plugins.plugins)
extra_immutable_input_digests = {
toolcp_relpath: tool_classpath.digest,
shimcp_relpath: shim_classfiles.digest,
plugins_relpath: merged_jvm_plugins_digest,
protoc_relpath: downloaded_protoc_binary.digest,
}
input_digest = await Get(
Digest,
MergeDigests([all_sources_stripped.snapshot.digest, empty_output_dir]))
result = await Get(
ProcessResult,
JvmProcess(
jdk=jdk,
classpath_entries=[
*tool_classpath.classpath_entries(toolcp_relpath),
shimcp_relpath
],
argv=[
"org.pantsbuild.backend.scala.scalapb.ScalaPBShim",
f"--protoc={os.path.join(protoc_relpath, downloaded_protoc_binary.exe)}",
*maybe_jvm_plugins_setup_args,
f"--scala_out={output_dir}",
*maybe_jvm_plugins_output_args,
*target_sources_stripped.snapshot.files,
],
input_digest=input_digest,
extra_immutable_input_digests=extra_immutable_input_digests,
extra_nailgun_keys=extra_immutable_input_digests,
description=
f"Generating Scala sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
extra_env=inherit_env,
),
)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def prepare_python_sources(
request: PythonSourceFilesRequest,
union_membership: UnionMembership) -> PythonSourceFiles:
sources = await Get(
SourceFiles,
SourceFilesRequest(
(tgt.get(SourcesField) for tgt in request.targets),
for_sources_types=request.valid_sources_types,
enable_codegen=True,
),
)
missing_init_files = await Get(
AncestorFiles,
AncestorFilesRequest(input_files=sources.snapshot.files,
requested=("__init__.py", "__init__.pyi")),
)
init_injected = await Get(
Snapshot,
MergeDigests(
(sources.snapshot.digest, missing_init_files.snapshot.digest)))
# Codegen is able to generate code in any arbitrary location, unlike sources normally being
# rooted under the target definition. To determine source roots for these generated files, we
# cannot use the normal `SourceRootRequest.for_target()` and we instead must determine
# a source root for every individual generated file. So, we re-resolve the codegen sources here.
python_and_resources_targets = []
codegen_targets = []
for tgt in request.targets:
if tgt.has_field(PythonSourceField) or tgt.has_field(
ResourceSourceField):
python_and_resources_targets.append(tgt)
elif tgt.get(SourcesField).can_generate(
PythonSourceField,
union_membership) or tgt.get(SourcesField).can_generate(
ResourceSourceField, union_membership):
codegen_targets.append(tgt)
codegen_sources = await MultiGet(
Get(
HydratedSources,
HydrateSourcesRequest(
tgt.get(SourcesField),
for_sources_types=request.valid_sources_types,
enable_codegen=True,
),
) for tgt in codegen_targets)
source_root_requests = [
*(SourceRootRequest.for_target(tgt)
for tgt in python_and_resources_targets),
*(SourceRootRequest.for_file(f) for sources in codegen_sources
for f in sources.snapshot.files),
]
source_root_objs = await MultiGet(
Get(SourceRoot, SourceRootRequest, req)
for req in source_root_requests)
source_root_paths = {
source_root_obj.path
for source_root_obj in source_root_objs
}
return PythonSourceFiles(
SourceFiles(init_injected, sources.unrooted_files),
tuple(sorted(source_root_paths)))
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest,
protoc: Protoc,
grpc_python_plugin: GrpcPythonPlugin,
python_protobuf_subsystem: PythonProtobufSubsystem,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSources, ),
),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSources]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
protoc_gen_mypy_script = "protoc-gen-mypy"
mypy_pex = None
if python_protobuf_subsystem.mypy_plugin:
mypy_pex = await Get(
VenvPex,
VenvPexRequest(
bin_names=[protoc_gen_mypy_script],
pex_request=PexRequest(
output_filename="mypy_protobuf.pex",
internal_only=True,
requirements=PexRequirements(
[python_protobuf_subsystem.mypy_plugin_version]),
# TODO(John Sirois): Fix these interpreter constraints to track the actual
# python requirement of the mypy_plugin_version or else plumb an option for
# manually setting the constraint to track what mypy_plugin_version needs:
# https://github.com/pantsbuild/pants/issues/11565
# Here we guess a constraint that will likely work with any mypy_plugin_version
# selected.
interpreter_constraints=PexInterpreterConstraints(
["CPython>=3.5"]),
),
),
)
downloaded_grpc_plugin = (await Get(
DownloadedExternalTool,
ExternalToolRequest,
grpc_python_plugin.get_request(Platform.current),
) if request.protocol_target.get(ProtobufGrpcToggle).value else None)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
if mypy_pex:
unmerged_digests.append(mypy_pex.digest)
if downloaded_grpc_plugin:
unmerged_digests.append(downloaded_grpc_plugin.digest)
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--python_out", output_dir]
if mypy_pex:
argv.extend([
f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}",
"--mypy_out",
output_dir,
])
if downloaded_grpc_plugin:
argv.extend([
f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}",
"--grpc_out", output_dir
])
argv.extend(target_sources_stripped.snapshot.files)
result = await Get(
ProcessResult,
Process(
argv,
input_digest=input_digest,
description=
f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_library's source root.
source_root_request = SourceRootRequest.for_target(
request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def generate_go_from_protobuf(
request: GenerateGoFromProtobufRequest,
protoc: Protoc,
go_protoc_plugin: _SetupGoProtocPlugin,
) -> GeneratedSources:
output_dir = "_generated_files"
protoc_relpath = "__protoc"
protoc_go_plugin_relpath = "__protoc_gen_go"
downloaded_protoc_binary, empty_output_dir, transitive_targets = await MultiGet(
Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(output_dir)])),
Get(TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address])),
)
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_sources_stripped, target_sources_stripped = await MultiGet(
Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
),
Get(StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]
])),
)
input_digest = await Get(
Digest,
MergeDigests([all_sources_stripped.snapshot.digest, empty_output_dir]))
maybe_grpc_plugin_args = []
if request.protocol_target.get(ProtobufGrpcToggleField).value:
maybe_grpc_plugin_args = [
f"--go-grpc_out={output_dir}",
"--go-grpc_opt=paths=source_relative",
]
result = await Get(
ProcessResult,
Process(
argv=[
os.path.join(protoc_relpath, downloaded_protoc_binary.exe),
f"--plugin=go={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go')}",
f"--plugin=go-grpc={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go-grpc')}",
f"--go_out={output_dir}",
"--go_opt=paths=source_relative",
*maybe_grpc_plugin_args,
*target_sources_stripped.snapshot.files,
],
# Note: Necessary or else --plugin option needs absolute path.
env={"PATH": protoc_go_plugin_relpath},
input_digest=input_digest,
immutable_input_digests={
protoc_relpath: downloaded_protoc_binary.digest,
protoc_go_plugin_relpath: go_protoc_plugin.digest,
},
description=
f"Generating Go sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def resolve_pex_entry_point(
request: ResolvePexEntryPointRequest) -> ResolvedPexEntryPoint:
ep_val = request.entry_point_field.value
ep_alias = request.entry_point_field.alias
address = request.entry_point_field.address
# TODO: factor up some of this code between python_awslambda and pex_binary once `sources` is
# removed.
# This code is tricky, as we support several different schemes:
# 1) `<none>` or `<None>` => set to `None`.
# 2) `path.to.module` => preserve exactly.
# 3) `path.to.module:func` => preserve exactly.
# 4) `app.py` => convert into `path.to.app`.
# 5) `app.py:func` => convert into `path.to.app:func`.
# 6) `:func` => if there's a sources field, convert to `path.to.sources:func` (deprecated).
# 7) no entry point field, but `sources` field => convert to `path.to.sources` (deprecated).
# Handle deprecated cases #6 and #7, which are the only cases where the `sources` field matters
# for calculating the entry point.
if not ep_val or ep_val.startswith(":"):
binary_source_paths = await Get(
Paths, PathGlobs,
request.sources.path_globs(FilesNotFoundBehavior.error))
if len(binary_source_paths.files) != 1:
instructions_url = docs_url(
"python-package-goal#creating-a-pex-file-from-a-pex_binary-target"
)
if not ep_val:
raise InvalidFieldException(
f"The `{ep_alias}` field is not set for the target {address}. Run "
f"`./pants help pex_binary` for more information on how to set the field or "
f"see {instructions_url}.")
raise InvalidFieldException(
f"The `{ep_alias}` field for the target {address} is set to the short-hand value "
f"{repr(ep_val)}, but the `sources` field is not set. Pants requires the "
"`sources` field to expand the entry point to the normalized form "
f"`path.to.module:{ep_val}`. Please either set the `sources` field to exactly one "
f"file (deprecated) or set `{ep_alias}='my_file.py:{ep_val}'`. See "
f"{instructions_url}.")
entry_point_path = binary_source_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
stripped_source_path = os.path.relpath(entry_point_path,
source_root.path)
module_base, _ = os.path.splitext(stripped_source_path)
normalized_path = module_base.replace(os.path.sep, ".")
return ResolvedPexEntryPoint(
f"{normalized_path}{ep_val}" if ep_val else normalized_path)
# Case #1.
if ep_val in ("<none>", "<None>"):
return ResolvedPexEntryPoint(None)
path, _, func = ep_val.partition(":")
# If it's already a module (cases #2 and #3), simply use that. Otherwise, convert the file name
# into a module path (cases #4 and #5).
if not path.endswith(".py"):
return ResolvedPexEntryPoint(ep_val)
# Use the engine to validate that the file exists and that it resolves to only one file.
full_glob = os.path.join(address.spec_path, path)
entry_point_paths = await Get(
Paths,
PathGlobs(
[full_glob],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=
f"{address}'s `{request.entry_point_field.alias}` field",
),
)
# We will have already raised if the glob did not match, i.e. if there were no files. But
# we need to check if they used a file glob (`*` or `**`) that resolved to >1 file.
if len(entry_point_paths.files) != 1:
raise InvalidFieldException(
f"Multiple files matched for the `{ep_alias}` {repr(ep_val)} for the target "
f"{address}, but only one file expected. Are you using a glob, rather than a file "
f"name?\n\nAll matching files: {list(entry_point_paths.files)}.")
entry_point_path = entry_point_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
stripped_source_path = os.path.relpath(entry_point_path, source_root.path)
module_base, _ = os.path.splitext(stripped_source_path)
normalized_path = module_base.replace(os.path.sep, ".")
return ResolvedPexEntryPoint(
f"{normalized_path}:{func}" if func else normalized_path)
async def create_python_binary_run_request(
field_set: PythonBinaryFieldSet,
python_binary_defaults: PythonBinaryDefaults,
pex_env: PexEnvironment,
) -> RunRequest:
entry_point = field_set.entry_point.value
if entry_point is None:
binary_source_paths = await Get(
Paths, PathGlobs,
field_set.sources.path_globs(FilesNotFoundBehavior.error))
if len(binary_source_paths.files) != 1:
raise InvalidFieldException(
"No `entry_point` was set for the target "
f"{repr(field_set.address)}, so it must have exactly one source, but it has "
f"{len(binary_source_paths.files)}")
entry_point_path = binary_source_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
entry_point = PythonBinarySources.translate_source_file_to_entry_point(
os.path.relpath(entry_point_path, source_root.path))
transitive_targets = await Get(TransitiveTargets,
Addresses([field_set.address]))
# Note that we get an intermediate PexRequest here (instead of going straight to a Pex)
# so that we can get the interpreter constraints for use in runner_pex_request.
requirements_pex_request = await Get(
PexRequest,
PexFromTargetsRequest,
PexFromTargetsRequest.for_requirements([field_set.address],
internal_only=True),
)
requirements_request = Get(Pex, PexRequest, requirements_pex_request)
sources_request = Get(
PythonSourceFiles,
PythonSourceFilesRequest(transitive_targets.closure,
include_files=True))
output_filename = f"{field_set.address.target_name}.pex"
runner_pex_request = Get(
Pex,
PexRequest(
output_filename=output_filename,
interpreter_constraints=requirements_pex_request.
interpreter_constraints,
additional_args=field_set.generate_additional_args(
python_binary_defaults),
internal_only=True,
# Note that the entry point file is not in the Pex itself, but on the
# PEX_PATH. This works fine!
entry_point=entry_point,
),
)
requirements, sources, runner_pex = await MultiGet(requirements_request,
sources_request,
runner_pex_request)
merged_digest = await Get(
Digest,
MergeDigests([
requirements.digest, sources.source_files.snapshot.digest,
runner_pex.digest
]),
)
def in_chroot(relpath: str) -> str:
return os.path.join("{chroot}", relpath)
args = pex_env.create_argv(in_chroot(runner_pex.name),
python=runner_pex.python)
chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**pex_env.environment_dict(python_configured=runner_pex.python is not None),
"PEX_PATH":
in_chroot(requirements_pex_request.output_filename),
"PEX_EXTRA_SYS_PATH":
":".join(chrooted_source_roots),
}
return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
async def resolve_pex_entry_point(
request: ResolvePexEntryPointRequest) -> ResolvedPexEntryPoint:
ep_val = request.entry_point_field.value
ep_alias = request.entry_point_field.alias
address = request.entry_point_field.address
# TODO: factor up some of this code between python_awslambda and pex_binary once `sources` is
# removed.
# This code is tricky, as we support several different schemes:
# 1) `<none>` or `<None>` => set to `None`.
# 2) `path.to.module` => preserve exactly.
# 3) `path.to.module:func` => preserve exactly.
# 4) `app.py` => convert into `path.to.app`.
# 5) `app.py:func` => convert into `path.to.app:func`.
if ep_val is None:
instructions_url = docs_url(
"python-package-goal#creating-a-pex-file-from-a-pex_binary-target")
raise InvalidFieldException(
f"The `{ep_alias}` field is not set for the target {address}. Run "
f"`./pants help pex_binary` for more information on how to set the field or "
f"see {instructions_url}.")
# Case #1.
if ep_val.module in ("<none>", "<None>"):
return ResolvedPexEntryPoint(None)
# If it's already a module (cases #2 and #3), simply use that. Otherwise, convert the file name
# into a module path (cases #4 and #5).
if not ep_val.module.endswith(".py"):
return ResolvedPexEntryPoint(ep_val)
# Use the engine to validate that the file exists and that it resolves to only one file.
full_glob = os.path.join(address.spec_path, ep_val.module)
entry_point_paths = await Get(
Paths,
PathGlobs(
[full_glob],
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin=
f"{address}'s `{request.entry_point_field.alias}` field",
),
)
# We will have already raised if the glob did not match, i.e. if there were no files. But
# we need to check if they used a file glob (`*` or `**`) that resolved to >1 file.
if len(entry_point_paths.files) != 1:
raise InvalidFieldException(
f"Multiple files matched for the `{ep_alias}` {ep_val.spec!r} for the target "
f"{address}, but only one file expected. Are you using a glob, rather than a file "
f"name?\n\nAll matching files: {list(entry_point_paths.files)}.")
entry_point_path = entry_point_paths.files[0]
source_root = await Get(
SourceRoot,
SourceRootRequest,
SourceRootRequest.for_file(entry_point_path),
)
stripped_source_path = os.path.relpath(entry_point_path, source_root.path)
module_base, _ = os.path.splitext(stripped_source_path)
normalized_path = module_base.replace(os.path.sep, ".")
return ResolvedPexEntryPoint(
dataclasses.replace(ep_val, module=normalized_path))
async def generate_java_from_protobuf(
request: GenerateJavaFromProtobufRequest,
protoc: Protoc,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_source` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--java_out", output_dir]
argv.extend(target_sources_stripped.snapshot.files)
result = await Get(
ProcessResult,
Process(
argv,
input_digest=input_digest,
description=
f"Generating Java sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)