def VersionUpgradeXform(n, envoy_internal_shadow, file_proto, params):
"""Transform a FileDescriptorProto from vN[alpha\d] to v(N+1).
Args:
n: version N to upgrade from.
envoy_internal_shadow: generate a shadow for Envoy internal use containing deprecated fields.
file_proto: vN[alpha\d] FileDescriptorProto message.
params: plugin parameters.
Returns:
v(N+1) FileDescriptorProto message.
"""
# Load type database.
typedb = utils.GetTypeDb()
# If this isn't a proto in an upgraded package, return None.
if file_proto.name not in typedb.next_version_protos or not typedb.next_version_protos[
file_proto.name]:
return None
# Otherwise, this .proto needs upgrading, do it.
freeze = 'extra_args' in params and params['extra_args'] == 'freeze'
existing_pkg_version_status = file_proto.options.Extensions[
status_pb2.file_status].package_version_status
# Normally, we are generating the NEXT_MAJOR_VERSION_CANDIDATE. However, if
# freezing and previously this was the active major version, the migrated
# version is now the ACTIVE version.
if freeze and existing_pkg_version_status == status_pb2.ACTIVE:
package_version_status = status_pb2.ACTIVE
else:
package_version_status = status_pb2.NEXT_MAJOR_VERSION_CANDIDATE
return traverse.TraverseFile(
file_proto, UpgradeVisitor(n, typedb, envoy_internal_shadow, package_version_status))
def Plugin(output_descriptors):
"""Protoc plugin entry point.
This defines protoc plugin and manages the stdin -> stdout flow. An
api_proto_plugin is defined by the provided visitor.
See
http://www.expobrain.net/2015/09/13/create-a-plugin-for-google-protocol-buffer/
for further details on protoc plugin basics.
Args:
output_descriptors: a list of OutputDescriptors.
"""
request = plugin_pb2.CodeGeneratorRequest()
request.ParseFromString(sys.stdin.buffer.read())
response = plugin_pb2.CodeGeneratorResponse()
cprofile_enabled = os.getenv('CPROFILE_ENABLED')
# We use request.file_to_generate rather than request.file_proto here since we
# are invoked inside a Bazel aspect, each node in the DAG will be visited once
# by the aspect and we only want to generate docs for the current node.
for file_to_generate in request.file_to_generate:
# Find the FileDescriptorProto for the file we actually are generating.
file_proto = [
pf for pf in request.proto_file if pf.name == file_to_generate
][0]
if cprofile_enabled:
pr = cProfile.Profile()
pr.enable()
for od in output_descriptors:
f = response.file.add()
f.name = file_proto.name + od.output_suffix
# Don't run API proto plugins on things like WKT types etc.
if not file_proto.package.startswith('envoy.'):
continue
if request.HasField("parameter") and od.want_params:
params = dict(
param.split('=') for param in request.parameter.split(','))
xformed_proto = od.xform(file_proto, params)
visitor_factory = od.visitor_factory(params)
else:
xformed_proto = od.xform(file_proto)
visitor_factory = od.visitor_factory()
f.content = traverse.TraverseFile(
xformed_proto, visitor_factory) if xformed_proto else ''
if cprofile_enabled:
pr.disable()
stats_stream = io.StringIO()
ps = pstats.Stats(pr, stream=stats_stream).sort_stats(
os.getenv('CPROFILE_SORTBY', 'cumulative'))
stats_file = response.file.add()
stats_file.name = file_proto.name + '.profile'
ps.print_stats()
stats_file.content = stats_stream.getvalue()
# Also include the original FileDescriptorProto as text proto, this is
# useful when debugging.
descriptor_file = response.file.add()
descriptor_file.name = file_proto.name + ".descriptor.proto"
descriptor_file.content = str(file_proto)
sys.stdout.buffer.write(response.SerializeToString())
def Plugin(output_suffix, visitor):
"""Protoc plugin entry point.
This defines protoc plugin and manages the stdin -> stdout flow. An
api_proto_plugin is defined by the provided visitor.
See
http://www.expobrain.net/2015/09/13/create-a-plugin-for-google-protocol-buffer/
for further details on protoc plugin basics.
Args:
output_suffix: output files are generated alongside their corresponding
input .proto, with this filename suffix.
visitor: visitor.Visitor defining the business logic of the plugin.
"""
request = plugin_pb2.CodeGeneratorRequest()
request.ParseFromString(sys.stdin.buffer.read())
response = plugin_pb2.CodeGeneratorResponse()
cprofile_enabled = os.getenv('CPROFILE_ENABLED')
# We use request.file_to_generate rather than request.file_proto here since we
# are invoked inside a Bazel aspect, each node in the DAG will be visited once
# by the aspect and we only want to generate docs for the current node.
for file_to_generate in request.file_to_generate:
# Find the FileDescriptorProto for the file we actually are generating.
file_proto = [pf for pf in request.proto_file if pf.name == file_to_generate][0]
f = response.file.add()
f.name = file_proto.name + output_suffix
if cprofile_enabled:
pr = cProfile.Profile()
pr.enable()
# We don't actually generate any RST right now, we just string dump the
# input proto file descriptor into the output file.
f.content = traverse.TraverseFile(file_proto, visitor)
if cprofile_enabled:
pr.disable()
stats_stream = io.StringIO()
ps = pstats.Stats(pr,
stream=stats_stream).sort_stats(os.getenv('CPROFILE_SORTBY', 'cumulative'))
stats_file = response.file.add()
stats_file.name = file_proto.name + output_suffix + '.profile'
ps.print_stats()
stats_file.content = stats_stream.getvalue()
# Also include the original FileDescriptorProto as text proto, this is
# useful when debugging.
descriptor_file = response.file.add()
descriptor_file.name = file_proto.name + ".descriptor.proto"
descriptor_file.content = str(file_proto)
sys.stdout.buffer.write(response.SerializeToString())
def V3MigrationXform(file_proto):
"""Transform a FileDescriptorProto from v2[alpha\d] to v3alpha.
Args:
file_proto: v2[alpha\d] FileDescriptorProto message.
Returns:
v3 FileDescriptorProto message.
"""
# Load type database.
typedb = utils.LoadTypeDb()
# If this isn't a proto in an upgraded package, return None.
if file_proto.package not in typedb.next_version_packages or not typedb.next_version_packages[
file_proto.package]:
return None
# Otherwise, this .proto needs upgrading, do it.
return traverse.TraverseFile(file_proto, UpgradeVisitor(typedb))
def V3MigrationXform(envoy_internal_shadow, file_proto):
"""Transform a FileDescriptorProto from v2[alpha\d] to v3.
Args:
envoy_internal_shadow: generate a shadow for Envoy internal use containing deprecated fields.
file_proto: v2[alpha\d] FileDescriptorProto message.
Returns:
v3 FileDescriptorProto message.
"""
# Load type database.
typedb = utils.GetTypeDb()
# If this isn't a proto in an upgraded package, return None.
if file_proto.name not in typedb.next_version_protos or not typedb.next_version_protos[
file_proto.name]:
return None
# Otherwise, this .proto needs upgrading, do it.
return traverse.TraverseFile(file_proto,
UpgradeVisitor(typedb, envoy_internal_shadow))
def FormatHeaderFromFile(source_code_info, file_proto, empty_file):
"""Format proto header.
Args:
source_code_info: SourceCodeInfo object.
file_proto: FileDescriptorProto for file.
empty_file: are there no message/enum/service defs in file?
Returns:
Formatted proto header as a string.
"""
# Load the type database.
typedb = utils.GetTypeDb()
# Figure out type dependencies in this .proto.
types = Types()
text_format.Merge(traverse.TraverseFile(file_proto, type_whisperer.TypeWhispererVisitor()), types)
type_dependencies = sum([list(t.type_dependencies) for t in types.types.values()], [])
for service in file_proto.service:
for m in service.method:
type_dependencies.extend([m.input_type[1:], m.output_type[1:]])
# Determine the envoy/ import paths from type deps.
envoy_proto_paths = set(
typedb.types[t].proto_path
for t in type_dependencies
if t.startswith('envoy.') and typedb.types[t].proto_path != file_proto.name)
def CamelCase(s):
return ''.join(t.capitalize() for t in re.split('[\._]', s))
package_line = 'package %s;\n' % file_proto.package
file_block = '\n'.join(['syntax = "proto3";\n', package_line])
options = descriptor_pb2.FileOptions()
options.java_outer_classname = CamelCase(os.path.basename(file_proto.name))
options.java_multiple_files = True
options.java_package = 'io.envoyproxy.' + file_proto.package
# This is a workaround for C#/Ruby namespace conflicts between packages and
# objects, see https://github.com/envoyproxy/envoy/pull/3854.
# TODO(htuch): remove once v3 fixes this naming issue in
# https://github.com/envoyproxy/envoy/issues/8120.
if file_proto.package in ['envoy.api.v2.listener', 'envoy.api.v2.cluster']:
qualified_package = '.'.join(s.capitalize() for s in file_proto.package.split('.')) + 'NS'
options.csharp_namespace = qualified_package
options.ruby_package = qualified_package
if file_proto.service:
options.java_generic_services = True
if file_proto.options.HasExtension(migrate_pb2.file_migrate):
options.Extensions[migrate_pb2.file_migrate].CopyFrom(
file_proto.options.Extensions[migrate_pb2.file_migrate])
if file_proto.options.HasExtension(
status_pb2.file_status) and file_proto.package.endswith('alpha'):
options.Extensions[status_pb2.file_status].CopyFrom(
file_proto.options.Extensions[status_pb2.file_status])
if not empty_file:
options.Extensions[
status_pb2.file_status].package_version_status = file_proto.options.Extensions[
status_pb2.file_status].package_version_status
options_block = FormatOptions(options)
requires_versioning_import = any(
protoxform_options.GetVersioningAnnotation(m.options) for m in file_proto.message_type)
envoy_imports = list(envoy_proto_paths)
google_imports = []
infra_imports = []
misc_imports = []
public_imports = []
for idx, d in enumerate(file_proto.dependency):
if idx in file_proto.public_dependency:
public_imports.append(d)
continue
elif d.startswith('envoy/annotations') or d.startswith('udpa/annotations'):
infra_imports.append(d)
elif d.startswith('envoy/'):
# We ignore existing envoy/ imports, since these are computed explicitly
# from type_dependencies.
pass
elif d.startswith('google/'):
google_imports.append(d)
elif d.startswith('validate/'):
infra_imports.append(d)
elif d in ['udpa/annotations/versioning.proto', 'udpa/annotations/status.proto']:
# Skip, we decide to add this based on requires_versioning_import and options.
pass
else:
misc_imports.append(d)
if options.HasExtension(status_pb2.file_status):
infra_imports.append('udpa/annotations/status.proto')
if requires_versioning_import:
infra_imports.append('udpa/annotations/versioning.proto')
def FormatImportBlock(xs):
if not xs:
return ''
return FormatBlock('\n'.join(sorted('import "%s";' % x for x in set(xs) if x)))
def FormatPublicImportBlock(xs):
if not xs:
return ''
return FormatBlock('\n'.join(sorted('import public "%s";' % x for x in xs)))
import_block = '\n'.join(
map(FormatImportBlock, [envoy_imports, google_imports, misc_imports, infra_imports]))
import_block += '\n' + FormatPublicImportBlock(public_imports)
comment_block = FormatComments(source_code_info.file_level_comments)
return ''.join(map(FormatBlock, [file_block, import_block, options_block, comment_block]))
type_context.ExtendOneof(oneof_index, field.name))
fields += '%soneof %s {\n%s%s' % (oneof_leading_comment, oneof_proto.name,
oneof_trailing_comment, FormatOptions(
oneof_proto.options))
fields += FormatBlock(FormatField(type_context.ExtendField(index, field.name), field))
if oneof_index is not None:
fields += '}\n\n'
return '%smessage %s {\n%s%s%s%s%s%s\n}\n' % (leading_comment, msg_proto.name, trailing_comment,
formatted_options, formatted_enums,
formatted_msgs, reserved_fields, fields)
def VisitFile(self, file_proto, type_context, services, msgs, enums):
empty_file = len(services) == 0 and len(enums) == 0 and len(msgs) == 0
header = FormatHeaderFromFile(type_context.source_code_info, file_proto, empty_file)
formatted_services = FormatBlock('\n'.join(services))
formatted_enums = FormatBlock('\n'.join(enums))
formatted_msgs = FormatBlock('\n'.join(msgs))
return ClangFormat(header + formatted_services + formatted_enums + formatted_msgs)
if __name__ == '__main__':
proto_desc_path = sys.argv[1]
file_proto = descriptor_pb2.FileDescriptorProto()
input_text = pathlib.Path(proto_desc_path).read_text()
if not input_text:
sys.exit(0)
text_format.Merge(input_text, file_proto)
dst_path = pathlib.Path(sys.argv[2])
utils.LoadTypeDb(sys.argv[3])
dst_path.write_bytes(traverse.TraverseFile(file_proto, ProtoFormatVisitor()))
field))
if oneof_index is not None:
fields += '}\n\n'
return '%smessage %s {\n%s%s%s%s%s%s\n}\n' % (
leading_comment, msg_proto.name, trailing_comment,
formatted_options, formatted_enums, formatted_msgs,
reserved_fields, fields)
def VisitFile(self, file_proto, type_context, services, msgs, enums):
empty_file = len(services) == 0 and len(enums) == 0 and len(msgs) == 0
header = FormatHeaderFromFile(type_context.source_code_info,
file_proto, empty_file)
formatted_services = FormatBlock('\n'.join(services))
formatted_enums = FormatBlock('\n'.join(enums))
formatted_msgs = FormatBlock('\n'.join(msgs))
return ClangFormat(header + formatted_services + formatted_enums +
formatted_msgs)
if __name__ == '__main__':
proto_desc_path = sys.argv[1]
file_proto = descriptor_pb2.FileDescriptorProto()
input_text = pathlib.Path(proto_desc_path).read_text()
if not input_text:
sys.exit(0)
text_format.Merge(input_text, file_proto)
dst_path = pathlib.Path(sys.argv[2])
utils.LoadTypeDb(sys.argv[3])
dst_path.write_bytes(
traverse.TraverseFile(file_proto, ProtoFormatVisitor()))
def FormatHeaderFromFile(source_code_info, file_proto):
"""Format proto header.
Args:
source_code_info: SourceCodeInfo object.
file_proto: FileDescriptorProto for file.
Returns:
Formatted proto header as a string.
"""
# Load the type database.
typedb = utils.LoadTypeDb()
# Figure out type dependencies in this .proto.
types = Types()
text_format.Merge(
traverse.TraverseFile(file_proto,
type_whisperer.TypeWhispererVisitor()), types)
type_dependencies = sum(
[list(t.type_dependencies) for t in types.types.values()], [])
for service in file_proto.service:
for m in service.method:
type_dependencies.extend([m.input_type[1:], m.output_type[1:]])
# Determine the envoy/ import paths from type deps.
envoy_proto_paths = set(typedb.types[t].proto_path
for t in type_dependencies
if t.startswith('envoy.')
and typedb.types[t].proto_path != file_proto.name)
def CamelCase(s):
return ''.join(t.capitalize() for t in re.split('[\._]', s))
package_line = 'package %s;\n' % file_proto.package
file_block = '\n'.join(['syntax = "proto3";\n', package_line])
options = [
'option java_outer_classname = "%s";' %
CamelCase(os.path.basename(file_proto.name)),
'option java_multiple_files = true;',
'option java_package = "io.envoyproxy.%s";' % file_proto.package,
]
# This is a workaround for C#/Ruby namespace conflicts between packages and
# objects, see https://github.com/envoyproxy/envoy/pull/3854.
# TODO(htuch): remove once v3 fixes this naming issue in
# https://github.com/envoyproxy/envoy/issues/8120.
if file_proto.package in ['envoy.api.v2.listener', 'envoy.api.v2.cluster']:
qualified_package = '.'.join(
s.capitalize() for s in file_proto.package.split('.')) + 'NS'
options += [
'option csharp_namespace = "%s";' % qualified_package,
'option ruby_package = "%s";' % qualified_package,
]
if file_proto.service:
options += ['option java_generic_services = true;']
options_block = FormatBlock('\n'.join(options))
envoy_imports = list(envoy_proto_paths)
google_imports = []
infra_imports = []
misc_imports = []
for d in file_proto.dependency:
if d.startswith('envoy/'):
# We ignore existing envoy/ imports, since these are computed explicitly
# from type_dependencies.
pass
elif d.startswith('google/'):
google_imports.append(d)
elif d.startswith('validate/'):
infra_imports.append(d)
else:
misc_imports.append(d)
def FormatImportBlock(xs):
if not xs:
return ''
return FormatBlock('\n'.join(sorted('import "%s";' % x for x in xs)))
import_block = '\n'.join(
map(FormatImportBlock,
[envoy_imports, google_imports, misc_imports, infra_imports]))
comment_block = FormatComments(source_code_info.file_level_comments)
return ''.join(
map(FormatBlock,
[file_block, options_block, import_block, comment_block]))
