def test_service_python_modules_signature():
service = make_service_with_method_options(
in_fields=(
# type=5 is int, so nothing is added.
descriptor_pb2.FieldDescriptorProto(name='secs', type=5),
descriptor_pb2.FieldDescriptorProto(
name='d',
type=14, # enum
type_name='a.b.c.v2.D',
),
),
method_signature='secs,d',
)
# Ensure that the service will have the expected imports.
method = service.methods['DoBigThing']
imports = {i.ident.python_import for i in method.ref_types}
assert imports == {
imp.Import(package=('a', 'b', 'c'), module='v2'),
imp.Import(package=('foo', ), module='bar'),
imp.Import(package=('foo', ), module='baz'),
imp.Import(package=('foo', ), module='qux'),
imp.Import(package=('google', 'api_core'), module='operation'),
imp.Import(package=('google', 'api_core'), module='operation_async'),
}
def make_field(name: str = 'my_field',
number: int = 1,
repeated: bool = False,
message: wrappers.MessageType = None,
enum: wrappers.EnumType = None,
meta: metadata.Metadata = None,
**kwargs) -> wrappers.Field:
T = desc.FieldDescriptorProto.Type
if message:
kwargs.setdefault('type_name', str(message.meta.address))
kwargs['type'] = 'TYPE_MESSAGE'
elif enum:
kwargs.setdefault('type_name', str(enum.meta.address))
kwargs['type'] = 'TYPE_ENUM'
else:
kwargs.setdefault('type', T.Value('TYPE_BOOL'))
if isinstance(kwargs['type'], str):
kwargs['type'] = T.Value(kwargs['type'])
label = kwargs.pop('label', 3 if repeated else 1)
field_pb = desc.FieldDescriptorProto(name=name,
label=label,
number=number,
**kwargs)
return wrappers.Field(
field_pb=field_pb,
enum=enum,
message=message,
meta=meta or metadata.Metadata(),
)
def test_mock_value_recursive(mock_method, expected):
# The elaborate setup is an unfortunate requirement.
file_pb = descriptor_pb2.FileDescriptorProto(
name="turtle.proto",
package="animalia.chordata.v2",
message_type=(
descriptor_pb2.DescriptorProto(
# It's turtles all the way down ;)
name="Turtle",
field=(
descriptor_pb2.FieldDescriptorProto(
name="turtle",
type="TYPE_MESSAGE",
type_name=".animalia.chordata.v2.Turtle",
number=1,
),
),
),
),
)
my_api = api.API.build([file_pb], package="animalia.chordata.v2")
turtle_field = my_api.messages["animalia.chordata.v2.Turtle"].fields["turtle"]
# If not handled properly, this will run forever and eventually OOM.
actual = getattr(turtle_field, mock_method)
assert actual == expected
def make_field(*, message=None, enum=None, **kwargs) -> wrappers.Field:
kwargs.setdefault('name', 'my_field')
kwargs.setdefault('number', 1)
kwargs.setdefault('type',
descriptor_pb2.FieldDescriptorProto.Type.Value('TYPE_BOOL'),
)
field_pb = descriptor_pb2.FieldDescriptorProto(**kwargs)
return wrappers.Field(field_pb=field_pb, message=message, enum=enum)
def make_field(*, message=None, enum=None, **kwargs) -> wrappers.Field:
T = descriptor_pb2.FieldDescriptorProto.Type
kwargs.setdefault('name', 'my_field')
kwargs.setdefault('number', 1)
kwargs.setdefault('type', T.Value('TYPE_BOOL'))
if isinstance(kwargs['type'], str):
kwargs['type'] = T.Value(kwargs['type'])
field_pb = descriptor_pb2.FieldDescriptorProto(**kwargs)
return wrappers.Field(field_pb=field_pb, message=message, enum=enum)
def make_field_pb2(name: str, number: int,
type: int = 11, # 11 == message
type_name: str = None,
) -> descriptor_pb2.FieldDescriptorProto:
return descriptor_pb2.FieldDescriptorProto(
name=name,
number=number,
type=type,
type_name=type_name,
)
def test_service_python_modules_signature():
service = make_service_with_method_options(
in_fields=(
descriptor_pb2.FieldDescriptorProto(name='secs', type=5),
descriptor_pb2.FieldDescriptorProto(
name='d',
type=11, # message
type_name='a.b.c.v2.D',
),
),
method_signature=signature_pb2.MethodSignature(fields=['secs', 'd']),
)
# type=5 is int, so nothing is added.
assert service.python_modules == (
('a.b.c', 'v2_pb2'),
('foo', 'bar_pb2'),
('foo', 'baz_pb2'),
('foo', 'qux_pb2'),
('google.api_core', 'operation'),
)
def test_service_python_modules_signature():
service = make_service_with_method_options(
in_fields=(
descriptor_pb2.FieldDescriptorProto(name='secs', type=5),
descriptor_pb2.FieldDescriptorProto(
name='d',
type=14, # enum
type_name='a.b.c.v2.D',
),
),
method_signature='secs,d',
)
# type=5 is int, so nothing is added.
assert service.python_modules == (
imp.Import(package=('a', 'b', 'c'), module='v2'),
imp.Import(package=('foo',), module='bar'),
imp.Import(package=('foo',), module='baz'),
imp.Import(package=('foo',), module='qux'),
imp.Import(package=('google', 'api_core'), module='operation'),
)
def make_field(name: str, repeated: bool = False,
meta: metadata.Metadata = None, **kwargs) -> wrappers.Method:
field_pb = descriptor_pb2.FieldDescriptorProto(
name=name,
label=3 if repeated else 1,
**kwargs
)
return wrappers.Field(
field_pb=field_pb,
meta=meta or metadata.Metadata(),
)
def get_field() -> wrappers.Field:
field_pb = descriptor_pb2.FieldDescriptorProto(
name='my_field',
number=1,
type=descriptor_pb2.FieldDescriptorProto.Type.Value('TYPE_BOOL'),
)
return wrappers.Field(field_pb=field_pb, meta=Metadata(
address=Address(package=['foo', 'bar'], module='baz'),
documentation=descriptor_pb2.SourceCodeInfo.Location(
leading_comments='Lorem ipsum dolor set amet',
),
))
def make_field_pb2(name: str, number: int,
type: int = 11, # 11 == message
type_name: str = None,
oneof_index: int = None,
**kwargs,
) -> desc.FieldDescriptorProto:
return desc.FieldDescriptorProto(
name=name,
number=number,
type=type,
type_name=type_name,
oneof_index=oneof_index,
**kwargs,
)
def make_field(name: str,
repeated: bool = False,
message: wrappers.MessageType = None,
meta: metadata.Metadata = None,
**kwargs) -> wrappers.Method:
if message:
kwargs['type_name'] = str(message.meta.address)
field_pb = descriptor_pb2.FieldDescriptorProto(name=name,
label=3 if repeated else 1,
**kwargs)
return wrappers.Field(
field_pb=field_pb,
message=message,
meta=meta or metadata.Metadata(),
)
def descriptor(self):
"""Return the descriptor for the field."""
proto_type = self.proto_type
if not self._descriptor:
# Resolve the message type, if any, to a string.
type_name = None
if isinstance(self.message, str):
if not self.message.startswith(self.package):
self.message = '{package}.{name}'.format(
package=self.package,
name=self.message,
)
type_name = self.message
elif self.message:
if hasattr(self.message, 'DESCRIPTOR'):
type_name = self.message.DESCRIPTOR.full_name
else:
type_name = self.message.meta.full_name
elif self.enum:
# Nos decipiat.
#
# As far as the wire format is concerned, enums are int32s.
# Protocol buffers itself also only sends ints; the enum
# objects are simply helper classes for translating names
# and values and it is the user's job to resolve to an int.
#
# Therefore, the non-trivial effort of adding the actual
# enum descriptors seems to add little or no actual value.
#
# FIXME: Eventually, come back and put in the actual enum
# descriptors.
proto_type = ProtoType.INT32
# Set the descriptor.
self._descriptor = descriptor_pb2.FieldDescriptorProto(
name=self.name,
number=self.number,
label=3 if self.repeated else 1,
type=proto_type,
type_name=type_name,
json_name=self.json_name,
proto3_optional=self.optional,
)
# Return the descriptor.
return self._descriptor
def descriptor(self):
"""Return the descriptor for the field."""
if not self._descriptor:
# Resolve the message type, if any, to a string.
type_name = None
if isinstance(self.message, str):
if not self.message.startswith(self.package):
self.message = "{package}.{name}".format(
package=self.package,
name=self.message,
)
type_name = self.message
elif self.message:
type_name = (
self.message.DESCRIPTOR.full_name
if hasattr(self.message, "DESCRIPTOR")
else self.message._meta.full_name
)
elif isinstance(self.enum, str):
if not self.enum.startswith(self.package):
self.enum = "{package}.{name}".format(
package=self.package,
name=self.enum,
)
type_name = self.enum
elif self.enum:
type_name = (
self.enum.DESCRIPTOR.full_name
if hasattr(self.enum, "DESCRIPTOR")
else self.enum._meta.full_name
)
# Set the descriptor.
self._descriptor = descriptor_pb2.FieldDescriptorProto(
name=self.name,
number=self.number,
label=3 if self.repeated else 1,
type=self.proto_type,
type_name=type_name,
json_name=self.json_name,
proto3_optional=self.optional,
)
# Return the descriptor.
return self._descriptor
def make_field_pb2(
name: str,
number: int,
label: int,
proto_type: int,
type_name: str = None,
oneof_index: int = None,
proto3_optional: bool = False,
options: desc.FieldOptions = None,
**kwargs,
) -> desc.FieldDescriptorProto:
return desc.FieldDescriptorProto(
name=name,
number=number,
label=label,
type=proto_type,
type_name=type_name,
oneof_index=oneof_index,
proto3_optional=proto3_optional,
options=options,
**kwargs,
)
def test_map_field_name_disambiguation():
squid_file_pb = descriptor_pb2.FileDescriptorProto(
name="mollusc.proto",
package="animalia.mollusca.v2",
message_type=(descriptor_pb2.DescriptorProto(name="Mollusc", ), ),
)
method_types_file_pb = descriptor_pb2.FileDescriptorProto(
name="mollusc_service.proto",
package="animalia.mollusca.v2",
message_type=(
descriptor_pb2.DescriptorProto(
name="CreateMolluscRequest",
field=(
descriptor_pb2.FieldDescriptorProto(
name="mollusc",
type="TYPE_MESSAGE",
type_name=".animalia.mollusca.v2.Mollusc",
number=1,
),
descriptor_pb2.FieldDescriptorProto(
name="molluscs_map",
type="TYPE_MESSAGE",
number=2,
type_name=
".animalia.mollusca.v2.CreateMolluscRequest.MolluscsMapEntry",
label="LABEL_REPEATED",
),
),
nested_type=(
descriptor_pb2.DescriptorProto(
name="MolluscsMapEntry",
field=(
descriptor_pb2.FieldDescriptorProto(
name="key",
type="TYPE_STRING",
number=1,
),
descriptor_pb2.FieldDescriptorProto(
name="value",
type="TYPE_MESSAGE",
number=2,
# We use the same type for the map value as for
# the singleton above to better highlight the
# problem raised in
# https://github.com/googleapis/gapic-generator-python/issues/618.
# The module _is_ disambiguated for singleton
# fields but NOT for map fields.
type_name=".animalia.mollusca.v2.Mollusc"),
),
options=descriptor_pb2.MessageOptions(map_entry=True),
), ),
), ),
)
my_api = api.API.build(
file_descriptors=[squid_file_pb, method_types_file_pb], )
create = my_api.messages['animalia.mollusca.v2.CreateMolluscRequest']
mollusc = create.fields['mollusc']
molluscs_map = create.fields['molluscs_map']
mollusc_ident = str(mollusc.type.ident)
mollusc_map_ident = str(molluscs_map.message.fields['value'].type.ident)
# The same module used in the same place should have the same import alias.
# Because there's a "mollusc" name used, the import should be disambiguated.
assert mollusc_ident == mollusc_map_ident == "am_mollusc.Mollusc"
def validate_and_transform_request(
self, calling_form: types.CallingForm,
request: List[Mapping[str, str]]) -> List[TransformedRequest]:
"""Validates and transforms the "request" block from a sample config.
In the initial request, each dict has a "field" key that maps to a dotted
variable name, e.g. clam.shell.
The only required keys in each dict are "field" and value".
Optional keys are "input_parameter", "value_is_file". and "comment".
All values in the initial request are strings except for the value
for "value_is_file", which is a bool.
The TransformedRequest structure of the return value has three fields:
"base", "body", and "single", where "base" maps to the top level attribute name,
"body" maps to a list of subfield assignment definitions, and "single"
maps to a singleton attribute assignment structure with no "field" value.
The "field" attribute in the requests in a "body" list have their prefix stripped;
the request in a "single" attribute has no "field" attribute.
Note: gRPC API methods only take one parameter (ignoring client-side streaming).
The reason that GAPIC client library API methods may take multiple parameters
is a workaround to provide idiomatic protobuf support within python.
The different 'bases' are really attributes for the singular request parameter.
TODO: properly handle subfields, indexing, and so forth.
TODO: Add/transform to list repeated element fields.
Requires proto/method/message descriptors.
E.g. [{"field": "clam.shell", "value": "10 kg", "input_parameter": "shell"},
{"field": "clam.pearls", "value": "3"},
{"field": "squid.mantle", "value": "100 kg"},
{"field": "whelk", "value": "speckled"}]
->
[TransformedRequest(
base="clam",
body=[AttributeRequestSetup(field="shell",
value="10 kg",
input_parameter="shell"),
AttributeRequestSetup(field="pearls", value="3")],
single=None),
TransformedRequest(base="squid",
body=[AttributeRequestSetup(field="mantle",
value="100 kg")],
single=None),
TransformedRequest(base="whelk",
body=None,
single=AttributeRequestSetup(value="speckled))]
The transformation makes it easier to set up request parameters in jinja
because it doesn't have to engage in prefix detection, validation,
or aggregation logic.
Args:
request (list[dict{str:str}]): The request body from the sample config
Returns:
List[TransformedRequest]: The transformed request block.
Raises:
InvalidRequestSetup: If a dict in the request lacks a "field" key,
a "value" key, if there is an unexpected keyword,
or if more than one base parameter is given for
a client-side streaming calling form.
BadAttributeLookup: If a request field refers to a non-existent field
in the request message type.
"""
base_param_to_attrs: Dict[
str, List[AttributeRequestSetup]] = defaultdict(list)
for r in request:
duplicate = dict(r)
val = duplicate.get("value")
if not val:
raise types.InvalidRequestSetup(
"Missing keyword in request entry: 'value'")
field = duplicate.get("field")
if not field:
raise types.InvalidRequestSetup(
"Missing keyword in request entry: 'field'")
spurious_keywords = set(duplicate.keys()) - {
"value", "field", "value_is_file", "input_parameter", "comment"
}
if spurious_keywords:
raise types.InvalidRequestSetup(
"Spurious keyword(s) in request entry: {}".format(
", ".join(f"'{kword}'"
for kword in spurious_keywords)))
input_parameter = duplicate.get("input_parameter")
if input_parameter:
self._handle_lvalue(
input_parameter,
wrappers.Field(
field_pb=descriptor_pb2.FieldDescriptorProto()))
attr_chain = field.split(".")
base = self.request_type_
for i, attr_name in enumerate(attr_chain):
attr = base.fields.get(attr_name)
if not attr:
raise types.BadAttributeLookup(
"Method request type {} has no attribute: '{}'".format(
self.request_type_.type, attr_name))
if attr.message:
base = attr.message
elif attr.enum:
# A little bit hacky, but 'values' is a list, and this is the easiest
# way to verify that the value is a valid enum variant.
witness = any(e.name == val for e in attr.enum.values)
if not witness:
raise types.InvalidEnumVariant(
"Invalid variant for enum {}: '{}'".format(
attr, val))
# Python code can set protobuf enums from strings.
# This is preferable to adding the necessary import statement
# and requires less munging of the assigned value
duplicate["value"] = f"'{val}'"
break
else:
raise TypeError
if i != len(attr_chain) - 1:
# We broke out of the loop after processing an enum.
extra_attrs = ".".join(attr_chain[i:])
raise types.InvalidEnumVariant(
f"Attempted to reference attributes of enum value: '{extra_attrs}'"
)
if len(attr_chain) > 1:
duplicate["field"] = ".".join(attr_chain[1:])
else:
# Because of the way top level attrs get rendered,
# there can't be duplicates.
# This is admittedly a bit of a hack.
if attr_chain[0] in base_param_to_attrs:
raise types.InvalidRequestSetup(
"Duplicated top level field in request block: '{}'".
format(attr_chain[0]))
del duplicate["field"]
# Mypy isn't smart enough to handle dictionary unpacking,
# so disable it for the AttributeRequestSetup ctor call.
base_param_to_attrs[attr_chain[0]].append(
AttributeRequestSetup(**duplicate)) # type: ignore
client_streaming_forms = {
types.CallingForm.RequestStreamingClient,
types.CallingForm.RequestStreamingBidi,
}
if len(base_param_to_attrs
) > 1 and calling_form in client_streaming_forms:
raise types.InvalidRequestSetup(
"Too many base parameters for client side streaming form")
return [(TransformedRequest(base=key, body=val, single=None)
if val[0].field else TransformedRequest(
base=key, body=None, single=val[0]))
for key, val in base_param_to_attrs.items()]
def validate_and_transform_request(
self, calling_form: types.CallingForm,
request: List[Mapping[str, str]]) -> FullRequest:
"""Validates and transforms the "request" block from a sample config.
In the initial request, each dict has a "field" key that maps to a dotted
variable name, e.g. clam.shell.
The only required keys in each dict are "field" and value".
Optional keys are "input_parameter", "value_is_file". and "comment".
All values in the initial request are strings except for the value
for "value_is_file", which is a bool.
The TransformedRequest structure of the return value has four fields:
"base", "body", "single", and "pattern",
where "base" maps to the top level attribute name,
"body" maps to a list of subfield assignment definitions, "single"
maps to a singleton attribute assignment structure with no "field" value,
and "pattern" is a resource name pattern string if the request describes
resource name construction.
The "field" attribute in the requests in a "body" list have their prefix stripped;
the request in a "single" attribute has no "field" attribute.
Note: gRPC API methods only take one parameter (ignoring client-side streaming).
The reason that GAPIC client library API methods may take multiple parameters
is a workaround to provide idiomatic protobuf support within python.
The different 'bases' are really attributes for the singular request parameter.
TODO: properly handle subfields, indexing, and so forth.
TODO: Add/transform to list repeated element fields.
Requires proto/method/message descriptors.
E.g. [{"field": "clam.shell", "value": "10 kg", "input_parameter": "shell"},
{"field": "clam.pearls", "value": "3"},
{"field": "squid.mantle", "value": "100 kg"},
{"field": "whelk", "value": "speckled"}]
->
[TransformedRequest(
base="clam",
body=[AttributeRequestSetup(field="shell",
value="10 kg",
input_parameter="shell"),
AttributeRequestSetup(field="pearls", value="3")],
single=None),
TransformedRequest(base="squid",
body=[AttributeRequestSetup(field="mantle",
value="100 kg")],
single=None),
TransformedRequest(base="whelk",
body=None,
single=AttributeRequestSetup(value="speckled))]
The transformation makes it easier to set up request parameters in jinja
because it doesn't have to engage in prefix detection, validation,
or aggregation logic.
Args:
request (list[dict{str:str}]): The request body from the sample config
Returns:
List[TransformedRequest]: The transformed request block.
Raises:
InvalidRequestSetup: If a dict in the request lacks a "field" key,
a "value" key, if there is an unexpected keyword,
or if more than one base parameter is given for
a client-side streaming calling form.
BadAttributeLookup: If a request field refers to a non-existent field
in the request message type.
ResourceRequestMismatch: If a request attempts to describe both
attribute manipulation and resource name
construction.
"""
base_param_to_attrs: Dict[str,
RequestEntry] = defaultdict(RequestEntry)
for r in request:
r_dup = dict(r)
val = r_dup.get("value")
if not val:
raise types.InvalidRequestSetup(
"Missing keyword in request entry: 'value'")
field = r_dup.get("field")
if not field:
raise types.InvalidRequestSetup(
"Missing keyword in request entry: 'field'")
spurious_kwords = set(r_dup.keys()) - self.VALID_REQUEST_KWORDS
if spurious_kwords:
raise types.InvalidRequestSetup(
"Spurious keyword(s) in request entry: {}".format(
", ".join(f"'{kword}'" for kword in spurious_kwords)))
input_parameter = r_dup.get("input_parameter")
if input_parameter:
self._handle_lvalue(
input_parameter,
wrappers.Field(
field_pb=descriptor_pb2.FieldDescriptorProto()),
)
# The percentage sign is used for setting up resource based requests
percent_idx = field.find("%")
if percent_idx == -1:
base_param, attr = self._normal_request_setup(
base_param_to_attrs, val, r_dup, field)
request_entry = base_param_to_attrs.get(base_param)
if request_entry and request_entry.is_resource_request:
raise types.ResourceRequestMismatch(
f"Request setup mismatch for base: {base_param}")
base_param_to_attrs[base_param].attrs.append(attr)
else:
# It's a resource based request.
base_param, resource_attr = (
field[:percent_idx],
field[percent_idx + 1:],
)
request_entry = base_param_to_attrs.get(base_param)
if request_entry and not request_entry.is_resource_request:
raise types.ResourceRequestMismatch(
f"Request setup mismatch for base: {base_param}")
if not self.request_type_.fields.get(base_param):
raise types.BadAttributeLookup(
"Method request type {} has no attribute: '{}'".format(
self.request_type_, base_param))
r_dup["field"] = resource_attr
request_entry = base_param_to_attrs[base_param]
request_entry.is_resource_request = True
request_entry.attrs.append(
AttributeRequestSetup(**r_dup) # type: ignore
)
client_streaming_forms = {
types.CallingForm.RequestStreamingClient,
types.CallingForm.RequestStreamingBidi,
}
if len(base_param_to_attrs
) > 1 and calling_form in client_streaming_forms:
raise types.InvalidRequestSetup(
"Too many base parameters for client side streaming form")
# We can only flatten a collection of request parameters if they're a
# subset of the flattened fields of the method.
flattenable = self.flattenable_fields >= set(base_param_to_attrs)
return FullRequest(
request_list=[
TransformedRequest.build(
self.request_type_,
self.api_schema_,
key,
val.attrs,
val.is_resource_request,
) for key, val in base_param_to_attrs.items()
],
flattenable=False,
)