async def hydrate_dockerfile(
request: GenerateDockerfileRequest) -> GeneratedSources:
target = request.protocol_target
address = target.address
instructions = target[DockerImageInstructionsField].value
if instructions and request.protocol_sources.files:
raise InvalidFieldException(
f"The `{target.alias}` {address} provides both a Dockerfile with the `source` field, "
"and Dockerfile contents with the `instructions` field, which is not supported.\n\n"
"To fix, please either set `source=None` or `instructions=None`.")
if not (instructions or request.protocol_sources.files):
raise InvalidFieldException(
f"The `{target.alias}` {address} does not specify any Dockerfile.\n\n"
"Provide either the filename to a Dockerfile in your workspace as the `source` field "
"value, or the Dockerfile content to the `instructions` field.")
def dockerfile_path():
name_parts = [
"Dockerfile", address.target_name, address.generated_name
]
return os.path.join(address.spec_path,
".".join(filter(bool, name_parts)))
output = (await Get(
Snapshot,
CreateDigest((FileContent(dockerfile_path(), "\n".join(
[*instructions, ""]).encode()), )),
) if instructions else request.protocol_sources)
return GeneratedSources(output)
async def relocate_files(
request: RelocateFilesViaCodegenRequest) -> GeneratedSources:
# Unlike normal codegen, we operate the on the sources of the `files_targets` field, not the
# `sources` of the original `relocated_sources` target.
# TODO(#13086): Because we're using `Targets` instead of `UnexpandedTargets`, the
# `files` target generator gets replaced by its generated `file` targets. That replacement is
# necessary because we only hydrate sources for `FileSourcesField`, which is only for the
# `file` target. That's really subtle!
original_file_targets = await Get(
Targets,
UnparsedAddressInputs,
request.protocol_target.get(
RelocatedFilesOriginalTargetsField).to_unparsed_address_inputs(),
)
original_files_sources = await MultiGet(
Get(
HydratedSources,
HydrateSourcesRequest(tgt.get(SourcesField),
for_sources_types=(FileSourceField, )),
) for tgt in original_file_targets)
snapshot = await Get(
Snapshot,
MergeDigests(sources.snapshot.digest
for sources in original_files_sources))
src_val = request.protocol_target.get(RelocatedFilesSrcField).value
dest_val = request.protocol_target.get(RelocatedFilesDestField).value
if src_val:
snapshot = await Get(Snapshot, RemovePrefix(snapshot.digest, src_val))
if dest_val:
snapshot = await Get(Snapshot, AddPrefix(snapshot.digest, dest_val))
return GeneratedSources(snapshot)
async def generate_java_from_wsdl(
request: GenerateJavaFromWsdlRequest) -> GeneratedSources:
sources = await Get(
HydratedSources,
HydrateSourcesRequest(request.protocol_target[WsdlSourceField]))
target_package = request.protocol_target[JavaPackageField].value
compile_results = await MultiGet(
Get(
CompiledWsdlSource,
CompileWsdlSourceRequest(
sources.snapshot.digest,
path=path,
module=request.protocol_target[JavaModuleField].value,
package=target_package,
),
) for path in sources.snapshot.files)
merged_output_digests, source_root = await MultiGet(
Get(Digest, MergeDigests([r.output_digest for r in compile_results])),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(merged_output_digests, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, merged_output_digests))
return GeneratedSources(source_root_restored)
async def generate_python_from_thrift(
request: GeneratePythonFromThriftRequest,
thrift_python: ThriftPythonSubsystem,
) -> GeneratedSources:
result = await Get(
GeneratedThriftSources,
GenerateThriftSourcesRequest(
thrift_source_field=request.protocol_target[ThriftSourceField],
lang_id="py",
lang_options=thrift_python.gen_options,
lang_name="Python",
),
)
# We must add back the source root for Python imports to work properly. Note that the file
# paths will be different depending on whether `namespace py` was used. See the tests for
# examples.
source_root = await Get(
SourceRoot, SourceRootRequest, SourceRootRequest.for_target(request.protocol_target)
)
source_root_restored = (
await Get(Snapshot, AddPrefix(result.snapshot.digest, source_root.path))
if source_root.path != "."
else await Get(Snapshot, Digest, result.snapshot.digest)
)
return GeneratedSources(source_root_restored)
async def relocate_files(request: RelocateFilesViaCodegenRequest) -> GeneratedSources:
# Unlike normal codegen, we operate the on the sources of the `files_targets` field, not the
# `sources` of the original `relocated_sources` target.
# TODO(#10915): using `await Get(Addresses, UnparsedAddressInputs)` causes a graph failure.
original_files_targets = await MultiGet(
Get(
WrappedTarget,
AddressInput,
AddressInput.parse(v, relative_to=request.protocol_target.address.spec_path),
)
for v in (
request.protocol_target.get(RelocatedFilesOriginalTargets)
.to_unparsed_address_inputs()
.values
)
)
original_files_sources = await MultiGet(
Get(HydratedSources, HydrateSourcesRequest(wrapped_tgt.target.get(Sources)))
for wrapped_tgt in original_files_targets
)
snapshot = await Get(
Snapshot, MergeDigests(sources.snapshot.digest for sources in original_files_sources)
)
src_val = request.protocol_target.get(RelocatedFilesSrcField).value
dest_val = request.protocol_target.get(RelocatedFilesDestField).value
if src_val:
snapshot = await Get(Snapshot, RemovePrefix(snapshot.digest, src_val))
if dest_val:
snapshot = await Get(Snapshot, AddPrefix(snapshot.digest, dest_val))
return GeneratedSources(snapshot)
async def generate_fortran_from_avro(request: GenerateFortranFromAvroRequest) -> GeneratedSources:
protocol_files = request.protocol_sources.files
def generate_fortran(fp: str) -> FileContent:
parent = str(PurePath(fp).parent).replace("src/avro", "src/fortran")
file_name = f"{PurePath(fp).stem}.f95"
return FileContent(str(PurePath(parent, file_name)), b"Generated")
result = await Get[Snapshot](InputFilesContent([generate_fortran(fp) for fp in protocol_files]))
return GeneratedSources(result)
async def _hydrate_asset_source(
request: GenerateSourcesRequest) -> GeneratedSources:
target = request.protocol_target
source_field = target[AssetSourceField]
if isinstance(source_field.value, str):
return GeneratedSources(request.protocol_sources)
http_source = source_field.value
file_digest = FileDigest(http_source.sha256, http_source.len)
# NB: This just has to run, we don't actually need the result because we know the Digest's
# FileEntry metadata.
await Get(Digest, DownloadFile(http_source.url, file_digest))
snapshot = await Get(
Snapshot,
CreateDigest([
FileEntry(
path=source_field.file_path,
file_digest=file_digest,
)
]),
)
return GeneratedSources(snapshot)
async def run_shell_command(
request: GenerateFilesFromShellCommandRequest,
) -> GeneratedSources:
shell_command = request.protocol_target
result = await Get(ProcessResult, ShellCommandProcessRequest(shell_command))
if shell_command[ShellCommandLogOutputField].value:
if result.stdout:
logger.info(result.stdout.decode())
if result.stderr:
logger.warning(result.stderr.decode())
working_directory = shell_command.address.spec_path
output = await Get(Snapshot, AddPrefix(result.output_digest, working_directory))
return GeneratedSources(output)
async def generate_smalltalk_from_avro(
request: GenerateSmalltalkFromAvroRequest,
) -> GeneratedSources:
protocol_files = request.protocol_sources.files
# Many codegen implementations will need to look up a protocol target's dependencies in their
# rule. We add this here to ensure that this does not result in rule graph issues.
_ = await Get(TransitiveTargets, Addresses([request.protocol_target.address]))
def generate_fortran(fp: str) -> FileContent:
parent = str(PurePath(fp).parent).replace("src/avro", "src/smalltalk")
file_name = f"{PurePath(fp).stem}.st"
return FileContent(str(PurePath(parent, file_name)), b"Generated")
result = await Get(Snapshot, CreateDigest([generate_fortran(fp) for fp in protocol_files]))
return GeneratedSources(result)
async def generate_scala_from_thrift_with_scrooge(
request: GenerateScalaFromThriftRequest, ) -> GeneratedSources:
result = await Get(
GeneratedScroogeThriftSources,
GenerateScroogeThriftSourcesRequest(
thrift_source_field=request.protocol_target[ThriftSourceField],
lang_id="scala",
lang_name="Scala",
),
)
source_root = await Get(
SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target))
source_root_restored = (await Get(
Snapshot, AddPrefix(result.snapshot.digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, result.snapshot.digest))
return GeneratedSources(source_root_restored)
async def generate_java_from_avro(
request: GenerateJavaFromAvroRequest, ) -> GeneratedSources:
sources = await Get(
HydratedSources,
HydrateSourcesRequest(request.protocol_target[AvroSourceField]))
compile_results = await MultiGet(
Get(CompiledAvroSource,
CompileAvroSourceRequest(sources.snapshot.digest, path))
for path in sources.snapshot.files)
merged_output_digest, source_root = await MultiGet(
Get(Digest, MergeDigests([r.output_digest for r in compile_results])),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(merged_output_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, merged_output_digest))
return GeneratedSources(source_root_restored)
async def generate_java_from_thrift(
request: GenerateJavaFromThriftRequest,
thrift_java: ApacheThriftJavaSubsystem,
) -> GeneratedSources:
result = await Get(
GeneratedThriftSources,
GenerateThriftSourcesRequest(
thrift_source_field=request.protocol_target[ThriftSourceField],
lang_id="java",
lang_options=thrift_java.gen_options,
lang_name="Java",
),
)
source_root = await Get(
SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target))
source_root_restored = (await Get(
Snapshot, AddPrefix(result.snapshot.digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, result.snapshot.digest))
return GeneratedSources(source_root_restored)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest, protoc: Protoc
) -> GeneratedSources:
download_protoc_request = Get[DownloadedExternalTool](
ExternalToolRequest, protoc.get_request(Platform.current)
)
output_dir = "_generated_files"
# TODO(#9650): replace this with a proper intrinsic to create empty directories.
create_output_dir_request = Get[ProcessResult](
Process(
("/bin/mkdir", output_dir),
description=f"Create the directory {output_dir}",
output_directories=(output_dir,),
)
)
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get[TransitiveTargets](Addresses([request.protocol_target.address]))
all_sources_request = Get[SourceFiles](
AllSourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSources,),
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
strip_source_roots=True,
)
)
stripped_target_sources_request = Get[SourceFiles](
AllSourceFilesRequest([request.protocol_target[ProtobufSources]], strip_source_roots=True)
)
(
downloaded_protoc_binary,
create_output_dir_result,
all_sources,
stripped_target_sources,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_sources_request,
stripped_target_sources_request,
)
input_digest = await Get[Digest](
MergeDigests(
(
all_sources.snapshot.digest,
downloaded_protoc_binary.digest,
create_output_dir_result.output_digest,
)
)
)
result = await Get[ProcessResult](
Process(
(
downloaded_protoc_binary.exe,
"--python_out",
output_dir,
*stripped_target_sources.snapshot.files,
),
input_digest=input_digest,
description=f"Generating Python sources from {request.protocol_target.address}.",
output_directories=(output_dir,),
)
)
normalized_snapshot = await Get[Snapshot](RemovePrefix(result.output_digest, output_dir))
return GeneratedSources(normalized_snapshot)
async def generate_go_from_protobuf(
request: GenerateGoFromProtobufRequest,
protoc: Protoc,
go_protoc_plugin: _SetupGoProtocPlugin,
) -> GeneratedSources:
output_dir = "_generated_files"
protoc_relpath = "__protoc"
protoc_go_plugin_relpath = "__protoc_gen_go"
downloaded_protoc_binary, empty_output_dir, transitive_targets = await MultiGet(
Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current)),
Get(Digest, CreateDigest([Directory(output_dir)])),
Get(TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address])),
)
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_sources_stripped, target_sources_stripped = await MultiGet(
Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
),
Get(StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]
])),
)
input_digest = await Get(
Digest,
MergeDigests([all_sources_stripped.snapshot.digest, empty_output_dir]))
maybe_grpc_plugin_args = []
if request.protocol_target.get(ProtobufGrpcToggleField).value:
maybe_grpc_plugin_args = [
f"--go-grpc_out={output_dir}",
"--go-grpc_opt=paths=source_relative",
]
result = await Get(
ProcessResult,
Process(
argv=[
os.path.join(protoc_relpath, downloaded_protoc_binary.exe),
f"--plugin=go={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go')}",
f"--plugin=go-grpc={os.path.join('.', protoc_go_plugin_relpath, 'protoc-gen-go-grpc')}",
f"--go_out={output_dir}",
"--go_opt=paths=source_relative",
*maybe_grpc_plugin_args,
*target_sources_stripped.snapshot.files,
],
# Note: Necessary or else --plugin option needs absolute path.
env={"PATH": protoc_go_plugin_relpath},
input_digest=input_digest,
immutable_input_digests={
protoc_relpath: downloaded_protoc_binary.digest,
protoc_go_plugin_relpath: go_protoc_plugin.digest,
},
description=
f"Generating Go sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def gen2(_: Gen2Request) -> GeneratedSources:
result = await Get(
Snapshot, CreateDigest([FileContent("src/haskell/app.hs", b"10 * 4")]))
return GeneratedSources(result)
async def generate_java_from_protobuf(
request: GenerateJavaFromProtobufRequest,
protoc: Protoc,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_source` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--java_out", output_dir]
argv.extend(target_sources_stripped.snapshot.files)
result = await Get(
ProcessResult,
Process(
argv,
input_digest=input_digest,
description=
f"Generating Java sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest,
protoc: Protoc,
grpc_python_plugin: GrpcPythonPlugin,
python_protobuf_subsystem: PythonProtobufSubsystem,
python_protobuf_mypy_plugin: PythonProtobufMypyPlugin,
pex_environment: PexEnvironment,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
protoc_gen_mypy_script = "protoc-gen-mypy"
protoc_gen_mypy_grpc_script = "protoc-gen-mypy_grpc"
mypy_pex = None
mypy_request = PexRequest(
output_filename="mypy_protobuf.pex",
internal_only=True,
requirements=python_protobuf_mypy_plugin.pex_requirements(),
interpreter_constraints=python_protobuf_mypy_plugin.
interpreter_constraints,
)
if python_protobuf_subsystem.mypy_plugin:
mypy_pex = await Get(
VenvPex,
VenvPexRequest(bin_names=[protoc_gen_mypy_script],
pex_request=mypy_request),
)
if request.protocol_target.get(ProtobufGrpcToggleField).value:
mypy_info = await Get(PexResolveInfo, VenvPex, mypy_pex)
# In order to generate stubs for gRPC code, we need mypy-protobuf 2.0 or above.
if any(dist_info.project_name == "mypy-protobuf"
and dist_info.version.major >= 2
for dist_info in mypy_info):
# TODO: Use `pex_path` once VenvPex stores a Pex field.
mypy_pex = await Get(
VenvPex,
VenvPexRequest(
bin_names=[
protoc_gen_mypy_script, protoc_gen_mypy_grpc_script
],
pex_request=mypy_request,
),
)
downloaded_grpc_plugin = (await Get(
DownloadedExternalTool,
ExternalToolRequest,
grpc_python_plugin.get_request(Platform.current),
) if request.protocol_target.get(ProtobufGrpcToggleField).value else None)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
if mypy_pex:
unmerged_digests.append(mypy_pex.digest)
if downloaded_grpc_plugin:
unmerged_digests.append(downloaded_grpc_plugin.digest)
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--python_out", output_dir]
if mypy_pex:
argv.extend([
f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}",
"--mypy_out",
output_dir,
])
if downloaded_grpc_plugin:
argv.extend([
f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}",
"--grpc_out", output_dir
])
if mypy_pex and protoc_gen_mypy_grpc_script in mypy_pex.bin:
argv.extend([
f"--plugin=protoc-gen-mypy_grpc={mypy_pex.bin[protoc_gen_mypy_grpc_script].argv0}",
"--mypy_grpc_out",
output_dir,
])
argv.extend(target_sources_stripped.snapshot.files)
result = await Get(
ProcessResult,
Process(
argv,
input_digest=input_digest,
description=
f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
append_only_caches=pex_environment.in_sandbox(
working_directory=None).append_only_caches,
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_source's source root.
source_root_request = SourceRootRequest.for_target(
request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def gen1(_: Gen1Request) -> GeneratedSources:
result = await Get(
Snapshot, CreateDigest([FileContent("assets/README.md", b"Hello!")]))
return GeneratedSources(result)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest, protoc: Protoc
) -> GeneratedSources:
download_protoc_request = Get(
DownloadedExternalTool, ExternalToolRequest, protoc.get_request(Platform.current)
)
output_dir = "_generated_files"
# TODO(#9650): replace this with a proper intrinsic to create empty directories.
create_output_dir_request = Get(
ProcessResult,
Process(
("/bin/mkdir", output_dir),
description=f"Create the directory {output_dir}",
level=LogLevel.DEBUG,
output_directories=(output_dir,),
),
)
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(TransitiveTargets, Addresses([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSources,),
),
)
target_stripped_sources_request = Get(
StrippedSourceFiles, SourceFilesRequest([request.protocol_target[ProtobufSources]]),
)
(
downloaded_protoc_binary,
create_output_dir_result,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
input_digest = await Get(
Digest,
MergeDigests(
(
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
create_output_dir_result.output_digest,
)
),
)
result = await Get(
ProcessResult,
Process(
(
downloaded_protoc_binary.exe,
"--python_out",
output_dir,
*target_sources_stripped.snapshot.files,
),
input_digest=input_digest,
description=f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir,),
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_library's source root.
source_root_request = SourceRootRequest.for_target(request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (
await Get(Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "."
else await Get(Snapshot, Digest, normalized_digest)
)
return GeneratedSources(source_root_restored)
async def generate_python_from_setuptools_scm(
request: GeneratePythonFromSetuptoolsSCMRequest,
setuptools_scm: SetuptoolsSCM,
) -> GeneratedSources:
# A GitWorktreeRequest is uncacheable, so this enclosing rule will run every time its result
# is needed, meaning it will always return a result based on the current underlying git state.
maybe_git_worktree = await Get(MaybeGitWorktree, GitWorktreeRequest())
if not maybe_git_worktree.git_worktree:
raise VCSVersioningError(
softwrap(
f"""
Trying to determine the version for the {request.protocol_target.address} target at
{request.protocol_target.address}, but you are not running in a git worktree.
"""
)
)
# Generate the setuptools_scm config. We don't use any existing pyproject.toml config,
# because we don't want to let setuptools_scm itself write the output file. This is because
# it would do so relative to the --root, meaning it will write outside the sandbox,
# directly into the workspace, which is obviously not what we want.
# It's unfortunate that setuptools_scm does not have separate config for "where is the .git
# directory" and "where should I write output to".
config: dict[str, dict[str, dict[str, str]]] = {}
tool_config = config.setdefault("tool", {}).setdefault("setuptools_scm", {})
tag_regex = request.protocol_target[VersionTagRegexField].value
if tag_regex:
tool_config["tag_regex"] = tag_regex
config_path = "pyproject.synthetic.toml"
input_digest_get = Get(
Digest,
CreateDigest(
[
FileContent(config_path, toml.dumps(config).encode()),
]
),
)
setuptools_scm_pex_get = Get(VenvPex, PexRequest, setuptools_scm.to_pex_request())
setuptools_scm_pex, input_digest = await MultiGet(setuptools_scm_pex_get, input_digest_get)
argv = ["--root", str(maybe_git_worktree.git_worktree.worktree), "--config", config_path]
result = await Get(
ProcessResult,
VenvPexProcess(
setuptools_scm_pex,
argv=argv,
input_digest=input_digest,
description=f"Run setuptools_scm for {request.protocol_target.address.spec}",
level=LogLevel.INFO,
),
)
version = result.stdout.decode().strip()
write_to = cast(str, request.protocol_target[VersionGenerateToField].value)
write_to_template = cast(str, request.protocol_target[VersionTemplateField].value)
output_content = write_to_template.format(version=version)
output_snapshot = await Get(
Snapshot, CreateDigest([FileContent(write_to, output_content.encode())])
)
return GeneratedSources(output_snapshot)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest,
protoc: Protoc,
grpc_python_plugin: GrpcPythonPlugin,
python_protobuf_subsystem: PythonProtobufSubsystem,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSources, ),
),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSources]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
protoc_gen_mypy_script = "protoc-gen-mypy"
mypy_pex = None
if python_protobuf_subsystem.mypy_plugin:
mypy_pex = await Get(
VenvPex,
VenvPexRequest(
bin_names=[protoc_gen_mypy_script],
pex_request=PexRequest(
output_filename="mypy_protobuf.pex",
internal_only=True,
requirements=PexRequirements(
[python_protobuf_subsystem.mypy_plugin_version]),
# TODO(John Sirois): Fix these interpreter constraints to track the actual
# python requirement of the mypy_plugin_version or else plumb an option for
# manually setting the constraint to track what mypy_plugin_version needs:
# https://github.com/pantsbuild/pants/issues/11565
# Here we guess a constraint that will likely work with any mypy_plugin_version
# selected.
interpreter_constraints=PexInterpreterConstraints(
["CPython>=3.5"]),
),
),
)
downloaded_grpc_plugin = (await Get(
DownloadedExternalTool,
ExternalToolRequest,
grpc_python_plugin.get_request(Platform.current),
) if request.protocol_target.get(ProtobufGrpcToggle).value else None)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
if mypy_pex:
unmerged_digests.append(mypy_pex.digest)
if downloaded_grpc_plugin:
unmerged_digests.append(downloaded_grpc_plugin.digest)
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--python_out", output_dir]
if mypy_pex:
argv.extend([
f"--plugin=protoc-gen-mypy={mypy_pex.bin[protoc_gen_mypy_script].argv0}",
"--mypy_out",
output_dir,
])
if downloaded_grpc_plugin:
argv.extend([
f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}",
"--grpc_out", output_dir
])
argv.extend(target_sources_stripped.snapshot.files)
result = await Get(
ProcessResult,
Process(
argv,
input_digest=input_digest,
description=
f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_library's source root.
source_root_request = SourceRootRequest.for_target(
request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def generate_python_from_protobuf(
request: GeneratePythonFromProtobufRequest,
protoc: Protoc,
grpc_python_plugin: GrpcPythonPlugin,
python_protobuf_subsystem: PythonProtobufSubsystem,
) -> GeneratedSources:
download_protoc_request = Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current))
output_dir = "_generated_files"
create_output_dir_request = Get(Digest,
CreateDigest([Directory(output_dir)]))
# Protoc needs all transitive dependencies on `protobuf_libraries` to work properly. It won't
# actually generate those dependencies; it only needs to look at their .proto files to work
# with imports.
# TODO(#10917): Use TransitiveTargets instead of TransitiveTargetsLite.
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequestLite([request.protocol_target.address]))
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest(
(tgt.get(Sources) for tgt in transitive_targets.closure),
for_sources_types=(ProtobufSources, ),
),
)
target_stripped_sources_request = Get(
StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSources]]))
(
downloaded_protoc_binary,
empty_output_dir,
all_sources_stripped,
target_sources_stripped,
) = await MultiGet(
download_protoc_request,
create_output_dir_request,
all_stripped_sources_request,
target_stripped_sources_request,
)
# To run the MyPy Protobuf plugin, we first install it with Pex, then extract the wheels and
# point Protoc to the extracted wheels with its `--plugin` argument.
extracted_mypy_wheels = None
if python_protobuf_subsystem.mypy_plugin:
mypy_pex = await Get(
Pex,
PexRequest(
output_filename="mypy_protobuf.pex",
internal_only=True,
requirements=PexRequirements(
[python_protobuf_subsystem.mypy_plugin_version]),
# This is solely to ensure that we use an appropriate interpreter when resolving
# the distribution. We don't actually run the distribution directly with Python,
# as we extract out its binary.
interpreter_constraints=PexInterpreterConstraints(
["CPython>=3.5"]),
),
)
extracted_mypy_wheels = await Get(ExtractedPexDistributions, Pex,
mypy_pex)
downloaded_grpc_plugin = (await Get(
DownloadedExternalTool,
ExternalToolRequest,
grpc_python_plugin.get_request(Platform.current),
) if request.protocol_target.get(ProtobufGrcpToggle).value else None)
unmerged_digests = [
all_sources_stripped.snapshot.digest,
downloaded_protoc_binary.digest,
empty_output_dir,
]
if extracted_mypy_wheels:
unmerged_digests.append(extracted_mypy_wheels.digest)
if downloaded_grpc_plugin:
unmerged_digests.append(downloaded_grpc_plugin.digest)
input_digest = await Get(Digest, MergeDigests(unmerged_digests))
argv = [downloaded_protoc_binary.exe, "--python_out", output_dir]
if extracted_mypy_wheels:
mypy_plugin_path = next(
p for p in extracted_mypy_wheels.wheel_directory_paths
if p.startswith(".deps/mypy_protobuf-"))
argv.extend([
f"--plugin=protoc-gen-mypy={mypy_plugin_path}/bin/protoc-gen-mypy",
"--mypy_out",
output_dir,
])
if downloaded_grpc_plugin:
argv.extend([
f"--plugin=protoc-gen-grpc={downloaded_grpc_plugin.exe}",
"--grpc_out", output_dir
])
argv.extend(target_sources_stripped.snapshot.files)
env = {}
if extracted_mypy_wheels:
env["PYTHONPATH"] = ":".join(
extracted_mypy_wheels.wheel_directory_paths)
result = await Get(
ProcessResult,
Process(
argv,
env=env,
input_digest=input_digest,
description=
f"Generating Python sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
),
)
# We must do some path manipulation on the output digest for it to look like normal sources,
# including adding back a source root.
py_source_root = request.protocol_target.get(PythonSourceRootField).value
if py_source_root:
# Verify that the python source root specified by the target is in fact a source root.
source_root_request = SourceRootRequest(PurePath(py_source_root))
else:
# The target didn't specify a python source root, so use the protobuf_library's source root.
source_root_request = SourceRootRequest.for_target(
request.protocol_target)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest, source_root_request),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def generate_scala_from_protobuf(
request: GenerateScalaFromProtobufRequest,
protoc: Protoc,
scalapb: ScalaPBSubsystem,
shim_classfiles: ScalaPBShimCompiledClassfiles,
jdk: InternalJdk,
) -> GeneratedSources:
output_dir = "_generated_files"
toolcp_relpath = "__toolcp"
shimcp_relpath = "__shimcp"
plugins_relpath = "__plugins"
protoc_relpath = "__protoc"
lockfile_request = await Get(GenerateJvmLockfileFromTool,
ScalapbcToolLockfileSentinel())
(
downloaded_protoc_binary,
tool_classpath,
empty_output_dir,
transitive_targets,
inherit_env,
) = await MultiGet(
Get(DownloadedExternalTool, ExternalToolRequest,
protoc.get_request(Platform.current)),
Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)),
Get(Digest, CreateDigest([Directory(output_dir)])),
Get(TransitiveTargets,
TransitiveTargetsRequest([request.protocol_target.address])),
# Need PATH so that ScalaPB can invoke `mkfifo`.
Get(Environment, EnvironmentRequest(requested=["PATH"])),
)
# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
# for Protobuf imports to be discoverable.
all_sources_stripped, target_sources_stripped = await MultiGet(
Get(
StrippedSourceFiles,
SourceFilesRequest(tgt[ProtobufSourceField]
for tgt in transitive_targets.closure
if tgt.has_field(ProtobufSourceField)),
),
Get(StrippedSourceFiles,
SourceFilesRequest([request.protocol_target[ProtobufSourceField]
])),
)
merged_jvm_plugins_digest = EMPTY_DIGEST
maybe_jvm_plugins_setup_args: tuple[str, ...] = ()
maybe_jvm_plugins_output_args: tuple[str, ...] = ()
jvm_plugins = scalapb.jvm_plugins
if jvm_plugins:
materialized_jvm_plugins = await Get(
MaterializedJvmPlugins, MaterializeJvmPluginsRequest(jvm_plugins))
merged_jvm_plugins_digest = materialized_jvm_plugins.digest
maybe_jvm_plugins_setup_args = materialized_jvm_plugins.setup_args(
plugins_relpath)
maybe_jvm_plugins_output_args = tuple(
f"--{plugin.name}_out={output_dir}"
for plugin in materialized_jvm_plugins.plugins)
extra_immutable_input_digests = {
toolcp_relpath: tool_classpath.digest,
shimcp_relpath: shim_classfiles.digest,
plugins_relpath: merged_jvm_plugins_digest,
protoc_relpath: downloaded_protoc_binary.digest,
}
input_digest = await Get(
Digest,
MergeDigests([all_sources_stripped.snapshot.digest, empty_output_dir]))
result = await Get(
ProcessResult,
JvmProcess(
jdk=jdk,
classpath_entries=[
*tool_classpath.classpath_entries(toolcp_relpath),
shimcp_relpath
],
argv=[
"org.pantsbuild.backend.scala.scalapb.ScalaPBShim",
f"--protoc={os.path.join(protoc_relpath, downloaded_protoc_binary.exe)}",
*maybe_jvm_plugins_setup_args,
f"--scala_out={output_dir}",
*maybe_jvm_plugins_output_args,
*target_sources_stripped.snapshot.files,
],
input_digest=input_digest,
extra_immutable_input_digests=extra_immutable_input_digests,
extra_nailgun_keys=extra_immutable_input_digests,
description=
f"Generating Scala sources from {request.protocol_target.address}.",
level=LogLevel.DEBUG,
output_directories=(output_dir, ),
extra_env=inherit_env,
),
)
normalized_digest, source_root = await MultiGet(
Get(Digest, RemovePrefix(result.output_digest, output_dir)),
Get(SourceRoot, SourceRootRequest,
SourceRootRequest.for_target(request.protocol_target)),
)
source_root_restored = (await Get(
Snapshot, AddPrefix(normalized_digest, source_root.path))
if source_root.path != "." else await Get(
Snapshot, Digest, normalized_digest))
return GeneratedSources(source_root_restored)
async def hydrate_input_sources(
request: GenerateDockerContextFiles) -> GeneratedSources:
# We simply pass the files on, as-is
return GeneratedSources(request.protocol_sources)
async def run_shell_command(
request: GenerateFilesFromShellCommandRequest,
shell_setup: ShellSetup,
bash: BashBinary,
) -> GeneratedSources:
shell_command = request.protocol_target
working_directory = shell_command.address.spec_path
command = shell_command[ShellCommandCommandField].value
tools = shell_command[ShellCommandToolsField].value
outputs = shell_command[ShellCommandOutputsField].value or ()
if not command:
raise ValueError(
f"Missing `command` line in `shell_command` target {shell_command.address}."
)
if not tools:
raise ValueError(
f"Must provide any `tools` used by the `shell_command` {shell_command.address}."
)
env = await Get(Environment, EnvironmentRequest(["PATH"]))
search_path = shell_setup.executable_search_path(env)
tool_requests = [
BinaryPathRequest(
binary_name=tool,
search_path=search_path,
) for tool in {*tools, *["mkdir", "ln"]}
if tool not in BASH_BUILTIN_COMMANDS
]
tool_paths = await MultiGet(
Get(BinaryPaths, BinaryPathRequest, request)
for request in tool_requests)
command_env = {
"TOOLS":
" ".join(shlex.quote(tool.binary_name) for tool in tool_requests),
}
for binary, tool_request in zip(tool_paths, tool_requests):
if binary.first_path:
command_env[tool_request.binary_name] = binary.first_path.path
else:
raise BinaryNotFoundError.from_request(
tool_request,
rationale=
f"execute experimental_shell_command {shell_command.address}",
)
transitive_targets = await Get(
TransitiveTargets,
TransitiveTargetsRequest([shell_command.address]),
)
sources = await Get(
SourceFiles,
SourceFilesRequest(
sources_fields=[
tgt.get(Sources) for tgt in transitive_targets.dependencies
],
for_sources_types=(
Sources,
FilesSources,
),
enable_codegen=True,
),
)
output_files = [f for f in outputs if not f.endswith("/")]
output_directories = [d for d in outputs if d.endswith("/")]
if working_directory in sources.snapshot.dirs:
input_digest = sources.snapshot.digest
else:
work_dir = await Get(Digest,
CreateDigest([Directory(working_directory)]))
input_digest = await Get(
Digest, MergeDigests([sources.snapshot.digest, work_dir]))
# Setup bin_relpath dir with symlinks to all requested tools, so that we can use PATH.
bin_relpath = ".bin"
setup_tool_symlinks_script = ";".join(
dedent(f"""\
$mkdir -p {bin_relpath}
for tool in $TOOLS; do $ln -s ${{!tool}} {bin_relpath}/; done
export PATH="$PWD/{bin_relpath}"
""").split("\n"))
result = await Get(
ProcessResult,
Process(
argv=(bash.path, "-c", setup_tool_symlinks_script + command),
description=
f"Running experimental_shell_command {shell_command.address}",
env=command_env,
input_digest=input_digest,
output_directories=output_directories,
output_files=output_files,
working_directory=working_directory,
),
)
if shell_command[ShellCommandLogOutputField].value:
if result.stdout:
logger.info(result.stdout.decode())
if result.stderr:
logger.warning(result.stderr.decode())
output = await Get(Snapshot,
AddPrefix(result.output_digest, working_directory))
return GeneratedSources(output)
