def test_service_python_modules_signature(): service = make_service_with_method_options( in_fields=( # type=5 is int, so nothing is added. descriptor_pb2.FieldDescriptorProto(name='secs', type=5), descriptor_pb2.FieldDescriptorProto( name='d', type=14, # enum type_name='a.b.c.v2.D', ), ), method_signature='secs,d', ) # Ensure that the service will have the expected imports. method = service.methods['DoBigThing'] imports = {i.ident.python_import for i in method.ref_types} assert imports == { imp.Import(package=('a', 'b', 'c'), module='v2'), imp.Import(package=('foo', ), module='bar'), imp.Import(package=('foo', ), module='baz'), imp.Import(package=('foo', ), module='qux'), imp.Import(package=('google', 'api_core'), module='operation'), imp.Import(package=('google', 'api_core'), module='operation_async'), }
def make_field(name: str = 'my_field', number: int = 1, repeated: bool = False, message: wrappers.MessageType = None, enum: wrappers.EnumType = None, meta: metadata.Metadata = None, **kwargs) -> wrappers.Field: T = desc.FieldDescriptorProto.Type if message: kwargs.setdefault('type_name', str(message.meta.address)) kwargs['type'] = 'TYPE_MESSAGE' elif enum: kwargs.setdefault('type_name', str(enum.meta.address)) kwargs['type'] = 'TYPE_ENUM' else: kwargs.setdefault('type', T.Value('TYPE_BOOL')) if isinstance(kwargs['type'], str): kwargs['type'] = T.Value(kwargs['type']) label = kwargs.pop('label', 3 if repeated else 1) field_pb = desc.FieldDescriptorProto(name=name, label=label, number=number, **kwargs) return wrappers.Field( field_pb=field_pb, enum=enum, message=message, meta=meta or metadata.Metadata(), )
def test_mock_value_recursive(mock_method, expected): # The elaborate setup is an unfortunate requirement. file_pb = descriptor_pb2.FileDescriptorProto( name="turtle.proto", package="animalia.chordata.v2", message_type=( descriptor_pb2.DescriptorProto( # It's turtles all the way down ;) name="Turtle", field=( descriptor_pb2.FieldDescriptorProto( name="turtle", type="TYPE_MESSAGE", type_name=".animalia.chordata.v2.Turtle", number=1, ), ), ), ), ) my_api = api.API.build([file_pb], package="animalia.chordata.v2") turtle_field = my_api.messages["animalia.chordata.v2.Turtle"].fields["turtle"] # If not handled properly, this will run forever and eventually OOM. actual = getattr(turtle_field, mock_method) assert actual == expected
def make_field(*, message=None, enum=None, **kwargs) -> wrappers.Field: kwargs.setdefault('name', 'my_field') kwargs.setdefault('number', 1) kwargs.setdefault('type', descriptor_pb2.FieldDescriptorProto.Type.Value('TYPE_BOOL'), ) field_pb = descriptor_pb2.FieldDescriptorProto(**kwargs) return wrappers.Field(field_pb=field_pb, message=message, enum=enum)
def make_field(*, message=None, enum=None, **kwargs) -> wrappers.Field: T = descriptor_pb2.FieldDescriptorProto.Type kwargs.setdefault('name', 'my_field') kwargs.setdefault('number', 1) kwargs.setdefault('type', T.Value('TYPE_BOOL')) if isinstance(kwargs['type'], str): kwargs['type'] = T.Value(kwargs['type']) field_pb = descriptor_pb2.FieldDescriptorProto(**kwargs) return wrappers.Field(field_pb=field_pb, message=message, enum=enum)
def make_field_pb2(name: str, number: int, type: int = 11, # 11 == message type_name: str = None, ) -> descriptor_pb2.FieldDescriptorProto: return descriptor_pb2.FieldDescriptorProto( name=name, number=number, type=type, type_name=type_name, )
def test_service_python_modules_signature(): service = make_service_with_method_options( in_fields=( descriptor_pb2.FieldDescriptorProto(name='secs', type=5), descriptor_pb2.FieldDescriptorProto( name='d', type=11, # message type_name='a.b.c.v2.D', ), ), method_signature=signature_pb2.MethodSignature(fields=['secs', 'd']), ) # type=5 is int, so nothing is added. assert service.python_modules == ( ('a.b.c', 'v2_pb2'), ('foo', 'bar_pb2'), ('foo', 'baz_pb2'), ('foo', 'qux_pb2'), ('google.api_core', 'operation'), )
def test_service_python_modules_signature(): service = make_service_with_method_options( in_fields=( descriptor_pb2.FieldDescriptorProto(name='secs', type=5), descriptor_pb2.FieldDescriptorProto( name='d', type=14, # enum type_name='a.b.c.v2.D', ), ), method_signature='secs,d', ) # type=5 is int, so nothing is added. assert service.python_modules == ( imp.Import(package=('a', 'b', 'c'), module='v2'), imp.Import(package=('foo',), module='bar'), imp.Import(package=('foo',), module='baz'), imp.Import(package=('foo',), module='qux'), imp.Import(package=('google', 'api_core'), module='operation'), )
def make_field(name: str, repeated: bool = False, meta: metadata.Metadata = None, **kwargs) -> wrappers.Method: field_pb = descriptor_pb2.FieldDescriptorProto( name=name, label=3 if repeated else 1, **kwargs ) return wrappers.Field( field_pb=field_pb, meta=meta or metadata.Metadata(), )
def get_field() -> wrappers.Field: field_pb = descriptor_pb2.FieldDescriptorProto( name='my_field', number=1, type=descriptor_pb2.FieldDescriptorProto.Type.Value('TYPE_BOOL'), ) return wrappers.Field(field_pb=field_pb, meta=Metadata( address=Address(package=['foo', 'bar'], module='baz'), documentation=descriptor_pb2.SourceCodeInfo.Location( leading_comments='Lorem ipsum dolor set amet', ), ))
def make_field_pb2(name: str, number: int, type: int = 11, # 11 == message type_name: str = None, oneof_index: int = None, **kwargs, ) -> desc.FieldDescriptorProto: return desc.FieldDescriptorProto( name=name, number=number, type=type, type_name=type_name, oneof_index=oneof_index, **kwargs, )
def make_field(name: str, repeated: bool = False, message: wrappers.MessageType = None, meta: metadata.Metadata = None, **kwargs) -> wrappers.Method: if message: kwargs['type_name'] = str(message.meta.address) field_pb = descriptor_pb2.FieldDescriptorProto(name=name, label=3 if repeated else 1, **kwargs) return wrappers.Field( field_pb=field_pb, message=message, meta=meta or metadata.Metadata(), )
def descriptor(self): """Return the descriptor for the field.""" proto_type = self.proto_type if not self._descriptor: # Resolve the message type, if any, to a string. type_name = None if isinstance(self.message, str): if not self.message.startswith(self.package): self.message = '{package}.{name}'.format( package=self.package, name=self.message, ) type_name = self.message elif self.message: if hasattr(self.message, 'DESCRIPTOR'): type_name = self.message.DESCRIPTOR.full_name else: type_name = self.message.meta.full_name elif self.enum: # Nos decipiat. # # As far as the wire format is concerned, enums are int32s. # Protocol buffers itself also only sends ints; the enum # objects are simply helper classes for translating names # and values and it is the user's job to resolve to an int. # # Therefore, the non-trivial effort of adding the actual # enum descriptors seems to add little or no actual value. # # FIXME: Eventually, come back and put in the actual enum # descriptors. proto_type = ProtoType.INT32 # Set the descriptor. self._descriptor = descriptor_pb2.FieldDescriptorProto( name=self.name, number=self.number, label=3 if self.repeated else 1, type=proto_type, type_name=type_name, json_name=self.json_name, proto3_optional=self.optional, ) # Return the descriptor. return self._descriptor
def descriptor(self): """Return the descriptor for the field.""" if not self._descriptor: # Resolve the message type, if any, to a string. type_name = None if isinstance(self.message, str): if not self.message.startswith(self.package): self.message = "{package}.{name}".format( package=self.package, name=self.message, ) type_name = self.message elif self.message: type_name = ( self.message.DESCRIPTOR.full_name if hasattr(self.message, "DESCRIPTOR") else self.message._meta.full_name ) elif isinstance(self.enum, str): if not self.enum.startswith(self.package): self.enum = "{package}.{name}".format( package=self.package, name=self.enum, ) type_name = self.enum elif self.enum: type_name = ( self.enum.DESCRIPTOR.full_name if hasattr(self.enum, "DESCRIPTOR") else self.enum._meta.full_name ) # Set the descriptor. self._descriptor = descriptor_pb2.FieldDescriptorProto( name=self.name, number=self.number, label=3 if self.repeated else 1, type=self.proto_type, type_name=type_name, json_name=self.json_name, proto3_optional=self.optional, ) # Return the descriptor. return self._descriptor
def make_field_pb2( name: str, number: int, label: int, proto_type: int, type_name: str = None, oneof_index: int = None, proto3_optional: bool = False, options: desc.FieldOptions = None, **kwargs, ) -> desc.FieldDescriptorProto: return desc.FieldDescriptorProto( name=name, number=number, label=label, type=proto_type, type_name=type_name, oneof_index=oneof_index, proto3_optional=proto3_optional, options=options, **kwargs, )
def test_map_field_name_disambiguation(): squid_file_pb = descriptor_pb2.FileDescriptorProto( name="mollusc.proto", package="animalia.mollusca.v2", message_type=(descriptor_pb2.DescriptorProto(name="Mollusc", ), ), ) method_types_file_pb = descriptor_pb2.FileDescriptorProto( name="mollusc_service.proto", package="animalia.mollusca.v2", message_type=( descriptor_pb2.DescriptorProto( name="CreateMolluscRequest", field=( descriptor_pb2.FieldDescriptorProto( name="mollusc", type="TYPE_MESSAGE", type_name=".animalia.mollusca.v2.Mollusc", number=1, ), descriptor_pb2.FieldDescriptorProto( name="molluscs_map", type="TYPE_MESSAGE", number=2, type_name= ".animalia.mollusca.v2.CreateMolluscRequest.MolluscsMapEntry", label="LABEL_REPEATED", ), ), nested_type=( descriptor_pb2.DescriptorProto( name="MolluscsMapEntry", field=( descriptor_pb2.FieldDescriptorProto( name="key", type="TYPE_STRING", number=1, ), descriptor_pb2.FieldDescriptorProto( name="value", type="TYPE_MESSAGE", number=2, # We use the same type for the map value as for # the singleton above to better highlight the # problem raised in # https://github.com/googleapis/gapic-generator-python/issues/618. # The module _is_ disambiguated for singleton # fields but NOT for map fields. type_name=".animalia.mollusca.v2.Mollusc"), ), options=descriptor_pb2.MessageOptions(map_entry=True), ), ), ), ), ) my_api = api.API.build( file_descriptors=[squid_file_pb, method_types_file_pb], ) create = my_api.messages['animalia.mollusca.v2.CreateMolluscRequest'] mollusc = create.fields['mollusc'] molluscs_map = create.fields['molluscs_map'] mollusc_ident = str(mollusc.type.ident) mollusc_map_ident = str(molluscs_map.message.fields['value'].type.ident) # The same module used in the same place should have the same import alias. # Because there's a "mollusc" name used, the import should be disambiguated. assert mollusc_ident == mollusc_map_ident == "am_mollusc.Mollusc"
def validate_and_transform_request( self, calling_form: types.CallingForm, request: List[Mapping[str, str]]) -> List[TransformedRequest]: """Validates and transforms the "request" block from a sample config. In the initial request, each dict has a "field" key that maps to a dotted variable name, e.g. clam.shell. The only required keys in each dict are "field" and value". Optional keys are "input_parameter", "value_is_file". and "comment". All values in the initial request are strings except for the value for "value_is_file", which is a bool. The TransformedRequest structure of the return value has three fields: "base", "body", and "single", where "base" maps to the top level attribute name, "body" maps to a list of subfield assignment definitions, and "single" maps to a singleton attribute assignment structure with no "field" value. The "field" attribute in the requests in a "body" list have their prefix stripped; the request in a "single" attribute has no "field" attribute. Note: gRPC API methods only take one parameter (ignoring client-side streaming). The reason that GAPIC client library API methods may take multiple parameters is a workaround to provide idiomatic protobuf support within python. The different 'bases' are really attributes for the singular request parameter. TODO: properly handle subfields, indexing, and so forth. TODO: Add/transform to list repeated element fields. Requires proto/method/message descriptors. E.g. [{"field": "clam.shell", "value": "10 kg", "input_parameter": "shell"}, {"field": "clam.pearls", "value": "3"}, {"field": "squid.mantle", "value": "100 kg"}, {"field": "whelk", "value": "speckled"}] -> [TransformedRequest( base="clam", body=[AttributeRequestSetup(field="shell", value="10 kg", input_parameter="shell"), AttributeRequestSetup(field="pearls", value="3")], single=None), TransformedRequest(base="squid", body=[AttributeRequestSetup(field="mantle", value="100 kg")], single=None), TransformedRequest(base="whelk", body=None, single=AttributeRequestSetup(value="speckled))] The transformation makes it easier to set up request parameters in jinja because it doesn't have to engage in prefix detection, validation, or aggregation logic. Args: request (list[dict{str:str}]): The request body from the sample config Returns: List[TransformedRequest]: The transformed request block. Raises: InvalidRequestSetup: If a dict in the request lacks a "field" key, a "value" key, if there is an unexpected keyword, or if more than one base parameter is given for a client-side streaming calling form. BadAttributeLookup: If a request field refers to a non-existent field in the request message type. """ base_param_to_attrs: Dict[ str, List[AttributeRequestSetup]] = defaultdict(list) for r in request: duplicate = dict(r) val = duplicate.get("value") if not val: raise types.InvalidRequestSetup( "Missing keyword in request entry: 'value'") field = duplicate.get("field") if not field: raise types.InvalidRequestSetup( "Missing keyword in request entry: 'field'") spurious_keywords = set(duplicate.keys()) - { "value", "field", "value_is_file", "input_parameter", "comment" } if spurious_keywords: raise types.InvalidRequestSetup( "Spurious keyword(s) in request entry: {}".format( ", ".join(f"'{kword}'" for kword in spurious_keywords))) input_parameter = duplicate.get("input_parameter") if input_parameter: self._handle_lvalue( input_parameter, wrappers.Field( field_pb=descriptor_pb2.FieldDescriptorProto())) attr_chain = field.split(".") base = self.request_type_ for i, attr_name in enumerate(attr_chain): attr = base.fields.get(attr_name) if not attr: raise types.BadAttributeLookup( "Method request type {} has no attribute: '{}'".format( self.request_type_.type, attr_name)) if attr.message: base = attr.message elif attr.enum: # A little bit hacky, but 'values' is a list, and this is the easiest # way to verify that the value is a valid enum variant. witness = any(e.name == val for e in attr.enum.values) if not witness: raise types.InvalidEnumVariant( "Invalid variant for enum {}: '{}'".format( attr, val)) # Python code can set protobuf enums from strings. # This is preferable to adding the necessary import statement # and requires less munging of the assigned value duplicate["value"] = f"'{val}'" break else: raise TypeError if i != len(attr_chain) - 1: # We broke out of the loop after processing an enum. extra_attrs = ".".join(attr_chain[i:]) raise types.InvalidEnumVariant( f"Attempted to reference attributes of enum value: '{extra_attrs}'" ) if len(attr_chain) > 1: duplicate["field"] = ".".join(attr_chain[1:]) else: # Because of the way top level attrs get rendered, # there can't be duplicates. # This is admittedly a bit of a hack. if attr_chain[0] in base_param_to_attrs: raise types.InvalidRequestSetup( "Duplicated top level field in request block: '{}'". format(attr_chain[0])) del duplicate["field"] # Mypy isn't smart enough to handle dictionary unpacking, # so disable it for the AttributeRequestSetup ctor call. base_param_to_attrs[attr_chain[0]].append( AttributeRequestSetup(**duplicate)) # type: ignore client_streaming_forms = { types.CallingForm.RequestStreamingClient, types.CallingForm.RequestStreamingBidi, } if len(base_param_to_attrs ) > 1 and calling_form in client_streaming_forms: raise types.InvalidRequestSetup( "Too many base parameters for client side streaming form") return [(TransformedRequest(base=key, body=val, single=None) if val[0].field else TransformedRequest( base=key, body=None, single=val[0])) for key, val in base_param_to_attrs.items()]
def validate_and_transform_request( self, calling_form: types.CallingForm, request: List[Mapping[str, str]]) -> FullRequest: """Validates and transforms the "request" block from a sample config. In the initial request, each dict has a "field" key that maps to a dotted variable name, e.g. clam.shell. The only required keys in each dict are "field" and value". Optional keys are "input_parameter", "value_is_file". and "comment". All values in the initial request are strings except for the value for "value_is_file", which is a bool. The TransformedRequest structure of the return value has four fields: "base", "body", "single", and "pattern", where "base" maps to the top level attribute name, "body" maps to a list of subfield assignment definitions, "single" maps to a singleton attribute assignment structure with no "field" value, and "pattern" is a resource name pattern string if the request describes resource name construction. The "field" attribute in the requests in a "body" list have their prefix stripped; the request in a "single" attribute has no "field" attribute. Note: gRPC API methods only take one parameter (ignoring client-side streaming). The reason that GAPIC client library API methods may take multiple parameters is a workaround to provide idiomatic protobuf support within python. The different 'bases' are really attributes for the singular request parameter. TODO: properly handle subfields, indexing, and so forth. TODO: Add/transform to list repeated element fields. Requires proto/method/message descriptors. E.g. [{"field": "clam.shell", "value": "10 kg", "input_parameter": "shell"}, {"field": "clam.pearls", "value": "3"}, {"field": "squid.mantle", "value": "100 kg"}, {"field": "whelk", "value": "speckled"}] -> [TransformedRequest( base="clam", body=[AttributeRequestSetup(field="shell", value="10 kg", input_parameter="shell"), AttributeRequestSetup(field="pearls", value="3")], single=None), TransformedRequest(base="squid", body=[AttributeRequestSetup(field="mantle", value="100 kg")], single=None), TransformedRequest(base="whelk", body=None, single=AttributeRequestSetup(value="speckled))] The transformation makes it easier to set up request parameters in jinja because it doesn't have to engage in prefix detection, validation, or aggregation logic. Args: request (list[dict{str:str}]): The request body from the sample config Returns: List[TransformedRequest]: The transformed request block. Raises: InvalidRequestSetup: If a dict in the request lacks a "field" key, a "value" key, if there is an unexpected keyword, or if more than one base parameter is given for a client-side streaming calling form. BadAttributeLookup: If a request field refers to a non-existent field in the request message type. ResourceRequestMismatch: If a request attempts to describe both attribute manipulation and resource name construction. """ base_param_to_attrs: Dict[str, RequestEntry] = defaultdict(RequestEntry) for r in request: r_dup = dict(r) val = r_dup.get("value") if not val: raise types.InvalidRequestSetup( "Missing keyword in request entry: 'value'") field = r_dup.get("field") if not field: raise types.InvalidRequestSetup( "Missing keyword in request entry: 'field'") spurious_kwords = set(r_dup.keys()) - self.VALID_REQUEST_KWORDS if spurious_kwords: raise types.InvalidRequestSetup( "Spurious keyword(s) in request entry: {}".format( ", ".join(f"'{kword}'" for kword in spurious_kwords))) input_parameter = r_dup.get("input_parameter") if input_parameter: self._handle_lvalue( input_parameter, wrappers.Field( field_pb=descriptor_pb2.FieldDescriptorProto()), ) # The percentage sign is used for setting up resource based requests percent_idx = field.find("%") if percent_idx == -1: base_param, attr = self._normal_request_setup( base_param_to_attrs, val, r_dup, field) request_entry = base_param_to_attrs.get(base_param) if request_entry and request_entry.is_resource_request: raise types.ResourceRequestMismatch( f"Request setup mismatch for base: {base_param}") base_param_to_attrs[base_param].attrs.append(attr) else: # It's a resource based request. base_param, resource_attr = ( field[:percent_idx], field[percent_idx + 1:], ) request_entry = base_param_to_attrs.get(base_param) if request_entry and not request_entry.is_resource_request: raise types.ResourceRequestMismatch( f"Request setup mismatch for base: {base_param}") if not self.request_type_.fields.get(base_param): raise types.BadAttributeLookup( "Method request type {} has no attribute: '{}'".format( self.request_type_, base_param)) r_dup["field"] = resource_attr request_entry = base_param_to_attrs[base_param] request_entry.is_resource_request = True request_entry.attrs.append( AttributeRequestSetup(**r_dup) # type: ignore ) client_streaming_forms = { types.CallingForm.RequestStreamingClient, types.CallingForm.RequestStreamingBidi, } if len(base_param_to_attrs ) > 1 and calling_form in client_streaming_forms: raise types.InvalidRequestSetup( "Too many base parameters for client side streaming form") # We can only flatten a collection of request parameters if they're a # subset of the flattened fields of the method. flattenable = self.flattenable_fields >= set(base_param_to_attrs) return FullRequest( request_list=[ TransformedRequest.build( self.request_type_, self.api_schema_, key, val.attrs, val.is_resource_request, ) for key, val in base_param_to_attrs.items() ], flattenable=False, )