def test_build_factory():
proto = descriptor_pb2.FileDescriptorProto(
package='google.mollusc.v1alpha1')
old = naming.Naming.build(proto, opts=options.Options(old_naming=True))
assert old.versioned_module_name == 'mollusc.v1alpha1'
new = naming.Naming.build(proto, opts=options.Options())
assert new.versioned_module_name == 'mollusc_v1alpha1'
def build(cls,
file_descriptor: descriptor_pb2.FileDescriptorProto,
file_to_generate: bool,
naming: api_naming.Naming,
opts: options.Options = options.Options(),
prior_protos: Mapping[str, 'Proto'] = None,
load_services: bool = True) -> 'Proto':
"""Build and return a Proto instance.
Args:
file_descriptor (~.FileDescriptorProto): The protocol buffer
object describing the proto file.
file_to_generate (bool): Whether this is a file which is
to be directly generated, or a dependency.
naming (~.Naming): The :class:`~.Naming` instance associated
with the API.
prior_protos (~.Proto): Previous, already processed protos.
These are needed to look up messages in imported protos.
load_services (bool): Toggle whether the proto file should
load its services. Not doing so enables a two-pass fix for
LRO response and metadata types in certain situations.
"""
return _ProtoBuilder(file_descriptor,
file_to_generate=file_to_generate,
naming=naming,
opts=opts,
prior_protos=prior_protos or {},
load_services=load_services).proto
def test_cli_override_name_and_namespace_versionless():
FileDesc = descriptor_pb2.FileDescriptorProto
proto1 = FileDesc(package='google.translation')
n = naming.Naming.build(
proto1,
opts=options.Options(namespace=('google', 'cloud'), name='translate'),
)
assert n.namespace == ('Google', 'Cloud')
assert n.name == 'Translate'
assert not n.version
def test_cli_override_namespace_dotted():
FileDesc = descriptor_pb2.FileDescriptorProto
proto1 = FileDesc(package='google.spanner.v1')
n = naming.Naming.build(
proto1,
opts=options.Options(namespace=('google.cloud', )),
)
assert n.namespace == ('Google', 'Cloud')
assert n.name == 'Spanner'
assert n.version == 'v1'
def test_cli_override_name_underscores():
FileDesc = descriptor_pb2.FileDescriptorProto
proto1 = FileDesc(package='google.cloud.videointelligence.v1')
n = naming.Naming.build(
proto1,
opts=options.Options(name='video_intelligence'),
)
assert n.namespace == ('Google', 'Cloud')
assert n.name == 'Video Intelligence'
assert n.version == 'v1'
def build(
cls,
file_descriptors: Sequence[descriptor_pb2.FileDescriptorProto],
package: str = '',
opts: options.Options = options.Options()) -> 'API':
"""Build the internal API schema based on the request.
Args:
file_descriptors (Sequence[~.FileDescriptorProto]): A list of
:class:`~.FileDescriptorProto` objects describing the
API.
package (str): A protocol buffer package, as a string, for which
code should be explicitly generated (including subpackages).
Protos with packages outside this list are considered imports
rather than explicit targets.
opts (~.options.Options): CLI options passed to the generator.
"""
# Save information about the overall naming for this API.
naming = api_naming.Naming.build(*filter(
lambda fd: fd.package.startswith(package),
file_descriptors,
),
opts=opts)
# Iterate over each FileDescriptorProto and fill out a Proto
# object describing it, and save these to the instance.
protos: Dict[str, Proto] = {}
for fd in file_descriptors:
protos[fd.name] = _ProtoBuilder(
file_descriptor=fd,
file_to_generate=fd.package.startswith(package),
naming=naming,
opts=opts,
prior_protos=protos,
).proto
# Done; return the API.
return cls(naming=naming, all_protos=protos)
def __init__(self,
file_descriptor: descriptor_pb2.FileDescriptorProto,
file_to_generate: bool,
naming: api_naming.Naming,
opts: options.Options = options.Options(),
prior_protos: Mapping[str, Proto] = None,
load_services: bool = True):
self.proto_messages: Dict[str, wrappers.MessageType] = {}
self.proto_enums: Dict[str, wrappers.EnumType] = {}
self.proto_services: Dict[str, wrappers.Service] = {}
self.file_descriptor = file_descriptor
self.file_to_generate = file_to_generate
self.prior_protos = prior_protos or {}
self.opts = opts
# Iterate over the documentation and place it into a dictionary.
#
# The comments in protocol buffers are sorted by a concept called
# the "path", which is a sequence of integers described in more
# detail below; this code simply shifts from a list to a dict,
# with tuples of paths as the dictionary keys.
self.docs: Dict[Tuple[int, ...],
descriptor_pb2.SourceCodeInfo.Location] = {}
for location in file_descriptor.source_code_info.location:
self.docs[tuple(location.path)] = location
# Everything has an "address", which is the proto where the thing
# was declared.
#
# We put this together by a baton pass of sorts: everything in
# this file *starts with* this address, which is appended to
# for each item as it is loaded.
self.address = metadata.Address(
api_naming=naming,
module=file_descriptor.name.split('/')[-1][:-len('.proto')],
package=tuple(file_descriptor.package.split('.')),
)
# Now iterate over the FileDescriptorProto and pull out each of
# the messages, enums, and services.
#
# The hard-coded path keys sent here are based on how descriptor.proto
# works; it uses the proto message number of the pieces of each
# message (e.g. the hard-code `4` for `message_type` immediately
# below is because `repeated DescriptorProto message_type = 4;` in
# descriptor.proto itself).
self._load_children(file_descriptor.enum_type,
self._load_enum,
address=self.address,
path=(5, ))
self._load_children(file_descriptor.message_type,
self._load_message,
address=self.address,
path=(4, ))
# Edge case: Protocol buffers is not particularly picky about
# ordering, and it is possible that a message will have had a field
# referencing another message which appears later in the file
# (or itself, recursively).
#
# In this situation, we would not have come across the message yet,
# and the field would have its original textual reference to the
# message (`type_name`) but not its resolved message wrapper.
orphan_field_gen = (
(field.type_name.lstrip('.'), field)
for message in self.proto_messages.values()
for field in message.fields.values()
if field.type_name and not (field.message or field.enum))
for key, field in orphan_field_gen:
maybe_msg_type = self.proto_messages.get(key)
maybe_enum_type = self.proto_enums.get(key)
if maybe_msg_type:
object.__setattr__(field, 'message', maybe_msg_type)
elif maybe_enum_type:
object.__setattr__(field, 'enum', maybe_enum_type)
else:
raise TypeError(f"Unknown type referenced in "
f"{self.file_descriptor.name}: '{key}'")
# Only generate the service if this is a target file to be generated.
# This prevents us from generating common services (e.g. LRO) when
# they are being used as an import just to get types declared in the
# same files.
if file_to_generate and load_services:
self._load_children(file_descriptor.service,
self._load_service,
address=self.address,
path=(6, ))
def build(
cls,
file_descriptors: Sequence[descriptor_pb2.FileDescriptorProto],
package: str = '',
opts: options.Options = options.Options(),
prior_protos: Mapping[str, 'Proto'] = None,
) -> 'API':
"""Build the internal API schema based on the request.
Args:
file_descriptors (Sequence[~.FileDescriptorProto]): A list of
:class:`~.FileDescriptorProto` objects describing the
API.
package (str): A protocol buffer package, as a string, for which
code should be explicitly generated (including subpackages).
Protos with packages outside this list are considered imports
rather than explicit targets.
opts (~.options.Options): CLI options passed to the generator.
prior_protos (~.Proto): Previous, already processed protos.
These are needed to look up messages in imported protos.
Primarily used for testing.
"""
# Save information about the overall naming for this API.
naming = api_naming.Naming.build(*filter(
lambda fd: fd.package.startswith(package),
file_descriptors,
),
opts=opts)
def disambiguate_keyword_fname(full_path: str,
visited_names: Container[str]) -> str:
path, fname = os.path.split(full_path)
name, ext = os.path.splitext(fname)
if name in keyword.kwlist or full_path in visited_names:
name += "_"
full_path = os.path.join(path, name + ext)
if full_path in visited_names:
return disambiguate_keyword_fname(full_path, visited_names)
return full_path
# Iterate over each FileDescriptorProto and fill out a Proto
# object describing it, and save these to the instance.
#
# The first pass gathers messages and enums but NOT services or methods.
# This is a workaround for a limitation in protobuf annotations for
# long running operations: the annotations are strings that reference
# message types but do not require a proto import.
# This hack attempts to address a common case where API authors,
# not wishing to generate an 'unused import' warning,
# don't import the proto file defining the real response or metadata
# type into the proto file that defines an LRO.
# We just load all the APIs types first and then
# load the services and methods with the full scope of types.
pre_protos: Dict[str, Proto] = dict(prior_protos or {})
for fd in file_descriptors:
fd.name = disambiguate_keyword_fname(fd.name, pre_protos)
pre_protos[fd.name] = Proto.build(
file_descriptor=fd,
file_to_generate=fd.package.startswith(package),
naming=naming,
opts=opts,
prior_protos=pre_protos,
# Ugly, ugly hack.
load_services=False,
)
# Second pass uses all the messages and enums defined in the entire API.
# This allows LRO returning methods to see all the types in the API,
# bypassing the above missing import problem.
protos: Dict[str, Proto] = {
name: Proto.build(
file_descriptor=proto.file_pb2,
file_to_generate=proto.file_to_generate,
naming=naming,
opts=opts,
prior_protos=pre_protos,
)
for name, proto in pre_protos.items()
}
# Done; return the API.
return cls(naming=naming, all_protos=protos)
def build(
cls,
*file_descriptors: descriptor_pb2.FileDescriptorProto,
opts: options.Options = options.Options(),
) -> 'Naming':
"""Return a full Naming instance based on these file descriptors.
This is pieced together from the proto package names as well as the
``google.api.metadata`` file annotation. This information may be
present in one or many files; this method is tolerant as long as
the data does not conflict.
Args:
file_descriptors (Iterable[~.FileDescriptorProto]): A list of
file descriptor protos. This list should only include the
files actually targeted for output (not their imports).
Returns:
~.Naming: A :class:`~.Naming` instance which is provided to
templates as part of the :class:`~.API`.
Raises:
ValueError: If the provided file descriptors contain contradictory
information.
"""
# Determine the set of proto packages.
proto_packages = {fd.package for fd in file_descriptors}
root_package = os.path.commonprefix(tuple(proto_packages)).rstrip('.')
# Sanity check: If there is no common ground in the package,
# we are obviously in trouble.
if not root_package:
raise ValueError(
'The protos provided do not share a common root package. '
'Ensure that all explicitly-specified protos are for a '
'single API. '
f'The packages we got are: {", ".join(proto_packages)}')
# Define the valid regex to split the package.
#
# It is not necessary for the regex to be as particular about package
# name validity (e.g. avoiding .. or segments starting with numbers)
# because protoc is guaranteed to give us valid package names.
pattern = r'^((?P<namespace>[a-z0-9_.]+)\.)?(?P<name>[a-z0-9_]+)'
# Only require the version portion of the regex if the version is
# present.
#
# This code may look counter-intuitive (why not use ? to make it
# optional), but the engine's greediness routine will decide that
# the version is the name, which is not what we want.
version = r'\.(?P<version>v[0-9]+(p[0-9]+)?((alpha|beta)[0-9]+)?)'
if re.search(version, root_package):
pattern += version
# Okay, do the match
match = cast(Match, re.search(pattern=pattern,
string=root_package)).groupdict()
match['namespace'] = match['namespace'] or ''
package_info = cls(
name=match['name'].capitalize(),
namespace=tuple(
[i.capitalize() for i in match['namespace'].split('.') if i]),
product_name=match['name'].capitalize(),
proto_package=root_package,
version=match.get('version', ''),
)
# Sanity check: Ensure that the package directives all inferred
# the same information.
if not package_info.version and len(proto_packages) > 1:
raise ValueError('All protos must have the same proto package '
'up to and including the version.')
# If a naming information was provided on the CLI, override the naming
# value.
#
# We are liberal about what formats we take on the CLI; it will
# likely make sense to many users to use dot-separated namespaces and
# snake case, so handle that and do the right thing.
if opts.name:
package_info = dataclasses.replace(
package_info,
name=' '.join([
i.capitalize()
for i in opts.name.replace('_', ' ').split(' ')
]))
if opts.namespace:
package_info = dataclasses.replace(
package_info,
namespace=tuple([
# The join-and-split on "." here causes us to expand out
# dot notation that we may have been sent; e.g. a one-tuple
# with ('x.y',) will become a two-tuple: ('x', 'y')
i.capitalize() for i in '.'.join(opts.namespace).split('.')
]))
# Done; return the naming information.
return package_info
def test_services():
L = descriptor_pb2.SourceCodeInfo.Location
# Make a silly helper method to not repeat some of the structure.
def _n(method_name: str):
return {
'service': 'google.example.v2.FooService',
'method': method_name,
}
# Set up retry information.
opts = options.Options(
retry={
'methodConfig': [
{
'name': [_n('TimeoutableGetFoo')],
'timeout': '30s'
},
{
'name': [_n('RetryableGetFoo')],
'retryPolicy': {
'maxAttempts': 3,
'initialBackoff': '%dn' % 1e6,
'maxBackoff': '60s',
'backoffMultiplier': 1.5,
'retryableStatusCodes': ['UNAVAILABLE', 'ABORTED'],
}
},
]
})
# Set up messages for our RPC.
request_message_pb = make_message_pb2(name='GetFooRequest',
fields=(make_field_pb2(name='name',
type=9,
number=1), ))
response_message_pb = make_message_pb2(name='GetFooResponse', fields=())
# Set up the service with an RPC.
service_pb = descriptor_pb2.ServiceDescriptorProto(
name='FooService',
method=(
descriptor_pb2.MethodDescriptorProto(
name='GetFoo',
input_type='google.example.v2.GetFooRequest',
output_type='google.example.v2.GetFooResponse',
),
descriptor_pb2.MethodDescriptorProto(
name='TimeoutableGetFoo',
input_type='google.example.v2.GetFooRequest',
output_type='google.example.v2.GetFooResponse',
),
descriptor_pb2.MethodDescriptorProto(
name='RetryableGetFoo',
input_type='google.example.v2.GetFooRequest',
output_type='google.example.v2.GetFooResponse',
),
),
)
# Fake-document our fake stuff.
locations = (
L(path=(6, 0), leading_comments='This is the FooService service.'),
L(path=(6, 0, 2, 0), leading_comments='This is the GetFoo method.'),
L(path=(4, 0), leading_comments='This is the GetFooRequest message.'),
L(path=(4, 1), leading_comments='This is the GetFooResponse message.'),
)
# Finally, set up the file that encompasses these.
fdp = make_file_pb2(
name='test.proto',
package='google.example.v2',
messages=(request_message_pb, response_message_pb),
services=(service_pb, ),
locations=locations,
)
# Make the proto object.
proto = api.API.build(
[fdp],
'google.example.v2',
opts=opts,
).protos['test.proto']
# Establish that our data looks correct.
assert len(proto.services) == 1
assert len(proto.messages) == 2
service = proto.services['google.example.v2.FooService']
assert service.meta.doc == 'This is the FooService service.'
assert len(service.methods) == 3
method = service.methods['GetFoo']
assert method.meta.doc == 'This is the GetFoo method.'
assert isinstance(method.input, wrappers.MessageType)
assert isinstance(method.output, wrappers.MessageType)
assert method.input.name == 'GetFooRequest'
assert method.input.meta.doc == 'This is the GetFooRequest message.'
assert method.output.name == 'GetFooResponse'
assert method.output.meta.doc == 'This is the GetFooResponse message.'
assert not method.timeout
assert not method.retry
# Establish that the retry information on a timeout-able method also
# looks correct.
timeout_method = service.methods['TimeoutableGetFoo']
assert timeout_method.timeout == pytest.approx(30.0)
assert not timeout_method.retry
# Establish that the retry information on the retryable method also
# looks correct.
retry_method = service.methods['RetryableGetFoo']
assert retry_method.timeout is None
assert retry_method.retry.max_attempts == 3
assert retry_method.retry.initial_backoff == pytest.approx(0.001)
assert retry_method.retry.backoff_multiplier == pytest.approx(1.5)
assert retry_method.retry.max_backoff == pytest.approx(60.0)
assert retry_method.retry.retryable_exceptions == {
exceptions.ServiceUnavailable,
exceptions.Aborted,
}
