def make_field(name: str = 'my_field', number: int = 1, repeated: bool = False, message: wrappers.MessageType = None, enum: wrappers.EnumType = None, meta: metadata.Metadata = None, **kwargs) -> wrappers.Field: T = desc.FieldDescriptorProto.Type if message: kwargs.setdefault('type_name', str(message.meta.address)) kwargs['type'] = 'TYPE_MESSAGE' elif enum: kwargs.setdefault('type_name', str(enum.meta.address)) kwargs['type'] = 'TYPE_ENUM' else: kwargs.setdefault('type', T.Value('TYPE_BOOL')) if isinstance(kwargs['type'], str): kwargs['type'] = T.Value(kwargs['type']) label = kwargs.pop('label', 3 if repeated else 1) field_pb = desc.FieldDescriptorProto(name=name, label=label, number=number, **kwargs) return wrappers.Field( field_pb=field_pb, enum=enum, message=message, meta=meta or metadata.Metadata(), )
def get_message( dot_path: str, *, fields: typing.Tuple[desc.FieldDescriptorProto] = (), ) -> wrappers.MessageType: # Pass explicit None through (for lro_metadata). if dot_path is None: return None # Note: The `dot_path` here is distinct from the canonical proto path # because it includes the module, which the proto path does not. # # So, if trying to test the DescriptorProto message here, the path # would be google.protobuf.descriptor.DescriptorProto (whereas the proto # path is just google.protobuf.DescriptorProto). pieces = dot_path.split('.') pkg, module, name = pieces[:-2], pieces[-2], pieces[-1] return wrappers.MessageType( fields={ i.name: wrappers.Field( field_pb=i, enum=get_enum(i.type_name) if i.type_name else None, ) for i in fields }, nested_messages={}, nested_enums={}, message_pb=desc.DescriptorProto(name=name, field=fields), meta=metadata.Metadata(address=metadata.Address( name=name, package=tuple(pkg), module=module, )), )
def make_field(*, message=None, enum=None, **kwargs) -> wrappers.Field: kwargs.setdefault('name', 'my_field') kwargs.setdefault('number', 1) kwargs.setdefault('type', descriptor_pb2.FieldDescriptorProto.Type.Value('TYPE_BOOL'), ) field_pb = descriptor_pb2.FieldDescriptorProto(**kwargs) return wrappers.Field(field_pb=field_pb, message=message, enum=enum)
def make_field(*, message=None, enum=None, **kwargs) -> wrappers.Field: T = descriptor_pb2.FieldDescriptorProto.Type kwargs.setdefault('name', 'my_field') kwargs.setdefault('number', 1) kwargs.setdefault('type', T.Value('TYPE_BOOL')) if isinstance(kwargs['type'], str): kwargs['type'] = T.Value(kwargs['type']) field_pb = descriptor_pb2.FieldDescriptorProto(**kwargs) return wrappers.Field(field_pb=field_pb, message=message, enum=enum)
def make_field(name: str, repeated: bool = False, meta: metadata.Metadata = None, **kwargs) -> wrappers.Method: field_pb = descriptor_pb2.FieldDescriptorProto( name=name, label=3 if repeated else 1, **kwargs ) return wrappers.Field( field_pb=field_pb, meta=meta or metadata.Metadata(), )
def _get_fields( self, field_pbs: Sequence[descriptor_pb2.FieldDescriptorProto], address: metadata.Address, path: Tuple[int, ...], oneofs: Optional[Dict[str, wrappers.Oneof]] = None ) -> Dict[str, wrappers.Field]: """Return a dictionary of wrapped fields for the given message. Args: field_pbs (Sequence[~.descriptor_pb2.FieldDescriptorProto]): A sequence of protobuf field objects. address (~.metadata.Address): An address object denoting the location of these fields. path (Tuple[int]): The source location path thus far, as understood by ``SourceCodeInfo.Location``. Returns: Mapping[str, ~.wrappers.Field]: A ordered mapping of :class:`~.wrappers.Field` objects. """ # Iterate over the fields and collect them into a dictionary. # # The saving of the enum and message types rely on protocol buffers' # naming rules to trust that they will never collide. # # Note: If this field is a recursive reference to its own message, # then the message will not be in `api_messages` yet (because the # message wrapper is not yet created, because it needs this object # first) and this will be None. This case is addressed in the # `_load_message` method. answer: Dict[str, wrappers.Field] = collections.OrderedDict() for i, field_pb in enumerate(field_pbs): is_oneof = oneofs and field_pb.HasField('oneof_index') oneof_name = nth((oneofs or {}).keys(), field_pb.oneof_index) if is_oneof else None field = wrappers.Field( field_pb=field_pb, enum=self.api_enums.get(field_pb.type_name.lstrip('.')), message=self.api_messages.get(field_pb.type_name.lstrip('.')), meta=metadata.Metadata( address=address.child(field_pb.name, path + (i, )), documentation=self.docs.get(path + (i, ), self.EMPTY), ), oneof=oneof_name, ) answer[field.name] = field # Done; return the answer. return answer
def make_field(name: str, repeated: bool = False, message: wrappers.MessageType = None, meta: metadata.Metadata = None, **kwargs) -> wrappers.Method: if message: kwargs['type_name'] = str(message.meta.address) field_pb = descriptor_pb2.FieldDescriptorProto(name=name, label=3 if repeated else 1, **kwargs) return wrappers.Field( field_pb=field_pb, message=message, meta=meta or metadata.Metadata(), )
def _get_fields( self, field_pbs: List[descriptor_pb2.FieldDescriptorProto], address: metadata.Address, path: Tuple[int], ) -> Mapping[str, wrappers.Field]: """Return a dictionary of wrapped fields for the given message. Args: fields (Sequence[~.descriptor_pb2.FieldDescriptorProto]): A sequence of protobuf field objects. address (~.metadata.Address): An address object denoting the location of these fields. path (Tuple[int]): The source location path thus far, as understood by ``SourceCodeInfo.Location``. Returns: Mapping[str, ~.wrappers.Field]: A ordered mapping of :class:`~.wrappers.Field` objects. """ # Iterate over the fields and collect them into a dictionary. # # The saving of the enum and message types rely on protocol buffers' # naming rules to trust that they will never collide. # # Note: If this field is a recursive reference to its own message, # then the message will not be in `all_messages` yet (because the # message wrapper is not yet created, because it needs this object # first) and this will be None. This case is addressed in the # `_load_message` method. answer = collections.OrderedDict() for field_pb, i in zip(field_pbs, range(0, sys.maxsize)): answer[field_pb.name] = wrappers.Field( field_pb=field_pb, enum=self.all_enums.get(field_pb.type_name.lstrip('.')), message=self.all_messages.get(field_pb.type_name.lstrip('.')), meta=metadata.Metadata( address=address, documentation=self.docs.get(path + (i, ), self.EMPTY), ), ) # Done; return the answer. return answer
def validate_and_transform_request( self, calling_form: types.CallingForm, request: List[Mapping[str, str]]) -> FullRequest: """Validates and transforms the "request" block from a sample config. In the initial request, each dict has a "field" key that maps to a dotted variable name, e.g. clam.shell. The only required keys in each dict are "field" and value". Optional keys are "input_parameter", "value_is_file". and "comment". All values in the initial request are strings except for the value for "value_is_file", which is a bool. The TransformedRequest structure of the return value has four fields: "base", "body", "single", and "pattern", where "base" maps to the top level attribute name, "body" maps to a list of subfield assignment definitions, "single" maps to a singleton attribute assignment structure with no "field" value, and "pattern" is a resource name pattern string if the request describes resource name construction. The "field" attribute in the requests in a "body" list have their prefix stripped; the request in a "single" attribute has no "field" attribute. Note: gRPC API methods only take one parameter (ignoring client-side streaming). The reason that GAPIC client library API methods may take multiple parameters is a workaround to provide idiomatic protobuf support within python. The different 'bases' are really attributes for the singular request parameter. TODO: properly handle subfields, indexing, and so forth. TODO: Add/transform to list repeated element fields. Requires proto/method/message descriptors. E.g. [{"field": "clam.shell", "value": "10 kg", "input_parameter": "shell"}, {"field": "clam.pearls", "value": "3"}, {"field": "squid.mantle", "value": "100 kg"}, {"field": "whelk", "value": "speckled"}] -> [TransformedRequest( base="clam", body=[AttributeRequestSetup(field="shell", value="10 kg", input_parameter="shell"), AttributeRequestSetup(field="pearls", value="3")], single=None), TransformedRequest(base="squid", body=[AttributeRequestSetup(field="mantle", value="100 kg")], single=None), TransformedRequest(base="whelk", body=None, single=AttributeRequestSetup(value="speckled))] The transformation makes it easier to set up request parameters in jinja because it doesn't have to engage in prefix detection, validation, or aggregation logic. Args: request (list[dict{str:str}]): The request body from the sample config Returns: List[TransformedRequest]: The transformed request block. Raises: InvalidRequestSetup: If a dict in the request lacks a "field" key, a "value" key, if there is an unexpected keyword, or if more than one base parameter is given for a client-side streaming calling form. BadAttributeLookup: If a request field refers to a non-existent field in the request message type. ResourceRequestMismatch: If a request attempts to describe both attribute manipulation and resource name construction. """ base_param_to_attrs: Dict[str, RequestEntry] = defaultdict(RequestEntry) for r in request: r_dup = dict(r) val = r_dup.get("value") if not val: raise types.InvalidRequestSetup( "Missing keyword in request entry: 'value'") field = r_dup.get("field") if not field: raise types.InvalidRequestSetup( "Missing keyword in request entry: 'field'") spurious_kwords = set(r_dup.keys()) - self.VALID_REQUEST_KWORDS if spurious_kwords: raise types.InvalidRequestSetup( "Spurious keyword(s) in request entry: {}".format( ", ".join(f"'{kword}'" for kword in spurious_kwords))) input_parameter = r_dup.get("input_parameter") if input_parameter: self._handle_lvalue( input_parameter, wrappers.Field( field_pb=descriptor_pb2.FieldDescriptorProto()), ) # The percentage sign is used for setting up resource based requests percent_idx = field.find("%") if percent_idx == -1: base_param, attr = self._normal_request_setup( base_param_to_attrs, val, r_dup, field) request_entry = base_param_to_attrs.get(base_param) if request_entry and request_entry.is_resource_request: raise types.ResourceRequestMismatch( f"Request setup mismatch for base: {base_param}") base_param_to_attrs[base_param].attrs.append(attr) else: # It's a resource based request. base_param, resource_attr = ( field[:percent_idx], field[percent_idx + 1:], ) request_entry = base_param_to_attrs.get(base_param) if request_entry and not request_entry.is_resource_request: raise types.ResourceRequestMismatch( f"Request setup mismatch for base: {base_param}") if not self.request_type_.fields.get(base_param): raise types.BadAttributeLookup( "Method request type {} has no attribute: '{}'".format( self.request_type_, base_param)) r_dup["field"] = resource_attr request_entry = base_param_to_attrs[base_param] request_entry.is_resource_request = True request_entry.attrs.append( AttributeRequestSetup(**r_dup) # type: ignore ) client_streaming_forms = { types.CallingForm.RequestStreamingClient, types.CallingForm.RequestStreamingBidi, } if len(base_param_to_attrs ) > 1 and calling_form in client_streaming_forms: raise types.InvalidRequestSetup( "Too many base parameters for client side streaming form") # We can only flatten a collection of request parameters if they're a # subset of the flattened fields of the method. flattenable = self.flattenable_fields >= set(base_param_to_attrs) return FullRequest( request_list=[ TransformedRequest.build( self.request_type_, self.api_schema_, key, val.attrs, val.is_resource_request, ) for key, val in base_param_to_attrs.items() ], flattenable=False, )
def validate_and_transform_request( self, calling_form: types.CallingForm, request: List[Mapping[str, str]]) -> List[TransformedRequest]: """Validates and transforms the "request" block from a sample config. In the initial request, each dict has a "field" key that maps to a dotted variable name, e.g. clam.shell. The only required keys in each dict are "field" and value". Optional keys are "input_parameter", "value_is_file". and "comment". All values in the initial request are strings except for the value for "value_is_file", which is a bool. The TransformedRequest structure of the return value has three fields: "base", "body", and "single", where "base" maps to the top level attribute name, "body" maps to a list of subfield assignment definitions, and "single" maps to a singleton attribute assignment structure with no "field" value. The "field" attribute in the requests in a "body" list have their prefix stripped; the request in a "single" attribute has no "field" attribute. Note: gRPC API methods only take one parameter (ignoring client-side streaming). The reason that GAPIC client library API methods may take multiple parameters is a workaround to provide idiomatic protobuf support within python. The different 'bases' are really attributes for the singular request parameter. TODO: properly handle subfields, indexing, and so forth. TODO: Add/transform to list repeated element fields. Requires proto/method/message descriptors. E.g. [{"field": "clam.shell", "value": "10 kg", "input_parameter": "shell"}, {"field": "clam.pearls", "value": "3"}, {"field": "squid.mantle", "value": "100 kg"}, {"field": "whelk", "value": "speckled"}] -> [TransformedRequest( base="clam", body=[AttributeRequestSetup(field="shell", value="10 kg", input_parameter="shell"), AttributeRequestSetup(field="pearls", value="3")], single=None), TransformedRequest(base="squid", body=[AttributeRequestSetup(field="mantle", value="100 kg")], single=None), TransformedRequest(base="whelk", body=None, single=AttributeRequestSetup(value="speckled))] The transformation makes it easier to set up request parameters in jinja because it doesn't have to engage in prefix detection, validation, or aggregation logic. Args: request (list[dict{str:str}]): The request body from the sample config Returns: List[TransformedRequest]: The transformed request block. Raises: InvalidRequestSetup: If a dict in the request lacks a "field" key, a "value" key, if there is an unexpected keyword, or if more than one base parameter is given for a client-side streaming calling form. BadAttributeLookup: If a request field refers to a non-existent field in the request message type. """ base_param_to_attrs: Dict[ str, List[AttributeRequestSetup]] = defaultdict(list) for r in request: duplicate = dict(r) val = duplicate.get("value") if not val: raise types.InvalidRequestSetup( "Missing keyword in request entry: 'value'") field = duplicate.get("field") if not field: raise types.InvalidRequestSetup( "Missing keyword in request entry: 'field'") spurious_keywords = set(duplicate.keys()) - { "value", "field", "value_is_file", "input_parameter", "comment" } if spurious_keywords: raise types.InvalidRequestSetup( "Spurious keyword(s) in request entry: {}".format( ", ".join(f"'{kword}'" for kword in spurious_keywords))) input_parameter = duplicate.get("input_parameter") if input_parameter: self._handle_lvalue( input_parameter, wrappers.Field( field_pb=descriptor_pb2.FieldDescriptorProto())) attr_chain = field.split(".") base = self.request_type_ for i, attr_name in enumerate(attr_chain): attr = base.fields.get(attr_name) if not attr: raise types.BadAttributeLookup( "Method request type {} has no attribute: '{}'".format( self.request_type_.type, attr_name)) if attr.message: base = attr.message elif attr.enum: # A little bit hacky, but 'values' is a list, and this is the easiest # way to verify that the value is a valid enum variant. witness = any(e.name == val for e in attr.enum.values) if not witness: raise types.InvalidEnumVariant( "Invalid variant for enum {}: '{}'".format( attr, val)) # Python code can set protobuf enums from strings. # This is preferable to adding the necessary import statement # and requires less munging of the assigned value duplicate["value"] = f"'{val}'" break else: raise TypeError if i != len(attr_chain) - 1: # We broke out of the loop after processing an enum. extra_attrs = ".".join(attr_chain[i:]) raise types.InvalidEnumVariant( f"Attempted to reference attributes of enum value: '{extra_attrs}'" ) if len(attr_chain) > 1: duplicate["field"] = ".".join(attr_chain[1:]) else: # Because of the way top level attrs get rendered, # there can't be duplicates. # This is admittedly a bit of a hack. if attr_chain[0] in base_param_to_attrs: raise types.InvalidRequestSetup( "Duplicated top level field in request block: '{}'". format(attr_chain[0])) del duplicate["field"] # Mypy isn't smart enough to handle dictionary unpacking, # so disable it for the AttributeRequestSetup ctor call. base_param_to_attrs[attr_chain[0]].append( AttributeRequestSetup(**duplicate)) # type: ignore client_streaming_forms = { types.CallingForm.RequestStreamingClient, types.CallingForm.RequestStreamingBidi, } if len(base_param_to_attrs ) > 1 and calling_form in client_streaming_forms: raise types.InvalidRequestSetup( "Too many base parameters for client side streaming form") return [(TransformedRequest(base=key, body=val, single=None) if val[0].field else TransformedRequest( base=key, body=None, single=val[0])) for key, val in base_param_to_attrs.items()]