Example #1
0
def make_field(name: str = 'my_field',
               number: int = 1,
               repeated: bool = False,
               message: wrappers.MessageType = None,
               enum: wrappers.EnumType = None,
               meta: metadata.Metadata = None,
               **kwargs) -> wrappers.Field:
    T = desc.FieldDescriptorProto.Type

    if message:
        kwargs.setdefault('type_name', str(message.meta.address))
        kwargs['type'] = 'TYPE_MESSAGE'
    elif enum:
        kwargs.setdefault('type_name', str(enum.meta.address))
        kwargs['type'] = 'TYPE_ENUM'
    else:
        kwargs.setdefault('type', T.Value('TYPE_BOOL'))

    if isinstance(kwargs['type'], str):
        kwargs['type'] = T.Value(kwargs['type'])

    label = kwargs.pop('label', 3 if repeated else 1)
    field_pb = desc.FieldDescriptorProto(name=name,
                                         label=label,
                                         number=number,
                                         **kwargs)
    return wrappers.Field(
        field_pb=field_pb,
        enum=enum,
        message=message,
        meta=meta or metadata.Metadata(),
    )
Example #2
0
def get_message(
    dot_path: str,
    *,
    fields: typing.Tuple[desc.FieldDescriptorProto] = (),
) -> wrappers.MessageType:
    # Pass explicit None through (for lro_metadata).
    if dot_path is None:
        return None

    # Note: The `dot_path` here is distinct from the canonical proto path
    # because it includes the module, which the proto path does not.
    #
    # So, if trying to test the DescriptorProto message here, the path
    # would be google.protobuf.descriptor.DescriptorProto (whereas the proto
    # path is just google.protobuf.DescriptorProto).
    pieces = dot_path.split('.')
    pkg, module, name = pieces[:-2], pieces[-2], pieces[-1]

    return wrappers.MessageType(
        fields={
            i.name: wrappers.Field(
                field_pb=i,
                enum=get_enum(i.type_name) if i.type_name else None,
            )
            for i in fields
        },
        nested_messages={},
        nested_enums={},
        message_pb=desc.DescriptorProto(name=name, field=fields),
        meta=metadata.Metadata(address=metadata.Address(
            name=name,
            package=tuple(pkg),
            module=module,
        )),
    )
def make_field(*, message=None, enum=None, **kwargs) -> wrappers.Field:
    kwargs.setdefault('name', 'my_field')
    kwargs.setdefault('number', 1)
    kwargs.setdefault('type',
        descriptor_pb2.FieldDescriptorProto.Type.Value('TYPE_BOOL'),
    )
    field_pb = descriptor_pb2.FieldDescriptorProto(**kwargs)
    return wrappers.Field(field_pb=field_pb, message=message, enum=enum)
Example #4
0
def make_field(*, message=None, enum=None, **kwargs) -> wrappers.Field:
    T = descriptor_pb2.FieldDescriptorProto.Type
    kwargs.setdefault('name', 'my_field')
    kwargs.setdefault('number', 1)
    kwargs.setdefault('type', T.Value('TYPE_BOOL'))
    if isinstance(kwargs['type'], str):
        kwargs['type'] = T.Value(kwargs['type'])
    field_pb = descriptor_pb2.FieldDescriptorProto(**kwargs)
    return wrappers.Field(field_pb=field_pb, message=message, enum=enum)
def make_field(name: str, repeated: bool = False,
               meta: metadata.Metadata = None, **kwargs) -> wrappers.Method:
    field_pb = descriptor_pb2.FieldDescriptorProto(
        name=name,
        label=3 if repeated else 1,
        **kwargs
    )
    return wrappers.Field(
        field_pb=field_pb,
        meta=meta or metadata.Metadata(),
    )
Example #6
0
    def _get_fields(
        self,
        field_pbs: Sequence[descriptor_pb2.FieldDescriptorProto],
        address: metadata.Address,
        path: Tuple[int, ...],
        oneofs: Optional[Dict[str, wrappers.Oneof]] = None
    ) -> Dict[str, wrappers.Field]:
        """Return a dictionary of wrapped fields for the given message.

        Args:
            field_pbs (Sequence[~.descriptor_pb2.FieldDescriptorProto]): A
                sequence of protobuf field objects.
            address (~.metadata.Address): An address object denoting the
                location of these fields.
            path (Tuple[int]): The source location path thus far, as
                understood by ``SourceCodeInfo.Location``.

        Returns:
            Mapping[str, ~.wrappers.Field]: A ordered mapping of
                :class:`~.wrappers.Field` objects.
        """
        # Iterate over the fields and collect them into a dictionary.
        #
        # The saving of the enum and message types rely on protocol buffers'
        # naming rules to trust that they will never collide.
        #
        # Note: If this field is a recursive reference to its own message,
        # then the message will not be in `api_messages` yet (because the
        # message wrapper is not yet created, because it needs this object
        # first) and this will be None. This case is addressed in the
        # `_load_message` method.
        answer: Dict[str, wrappers.Field] = collections.OrderedDict()
        for i, field_pb in enumerate(field_pbs):
            is_oneof = oneofs and field_pb.HasField('oneof_index')
            oneof_name = nth((oneofs or {}).keys(),
                             field_pb.oneof_index) if is_oneof else None

            field = wrappers.Field(
                field_pb=field_pb,
                enum=self.api_enums.get(field_pb.type_name.lstrip('.')),
                message=self.api_messages.get(field_pb.type_name.lstrip('.')),
                meta=metadata.Metadata(
                    address=address.child(field_pb.name, path + (i, )),
                    documentation=self.docs.get(path + (i, ), self.EMPTY),
                ),
                oneof=oneof_name,
            )
            answer[field.name] = field

        # Done; return the answer.
        return answer
def make_field(name: str,
               repeated: bool = False,
               message: wrappers.MessageType = None,
               meta: metadata.Metadata = None,
               **kwargs) -> wrappers.Method:
    if message:
        kwargs['type_name'] = str(message.meta.address)
    field_pb = descriptor_pb2.FieldDescriptorProto(name=name,
                                                   label=3 if repeated else 1,
                                                   **kwargs)
    return wrappers.Field(
        field_pb=field_pb,
        message=message,
        meta=meta or metadata.Metadata(),
    )
    def _get_fields(
        self,
        field_pbs: List[descriptor_pb2.FieldDescriptorProto],
        address: metadata.Address,
        path: Tuple[int],
    ) -> Mapping[str, wrappers.Field]:
        """Return a dictionary of wrapped fields for the given message.

        Args:
            fields (Sequence[~.descriptor_pb2.FieldDescriptorProto]): A
                sequence of protobuf field objects.
            address (~.metadata.Address): An address object denoting the
                location of these fields.
            path (Tuple[int]): The source location path thus far, as
                understood by ``SourceCodeInfo.Location``.

        Returns:
            Mapping[str, ~.wrappers.Field]: A ordered mapping of
                :class:`~.wrappers.Field` objects.
        """
        # Iterate over the fields and collect them into a dictionary.
        #
        # The saving of the enum and message types rely on protocol buffers'
        # naming rules to trust that they will never collide.
        #
        # Note: If this field is a recursive reference to its own message,
        # then the message will not be in `all_messages` yet (because the
        # message wrapper is not yet created, because it needs this object
        # first) and this will be None. This case is addressed in the
        # `_load_message` method.
        answer = collections.OrderedDict()
        for field_pb, i in zip(field_pbs, range(0, sys.maxsize)):
            answer[field_pb.name] = wrappers.Field(
                field_pb=field_pb,
                enum=self.all_enums.get(field_pb.type_name.lstrip('.')),
                message=self.all_messages.get(field_pb.type_name.lstrip('.')),
                meta=metadata.Metadata(
                    address=address,
                    documentation=self.docs.get(path + (i, ), self.EMPTY),
                ),
            )

        # Done; return the answer.
        return answer
    def validate_and_transform_request(
            self, calling_form: types.CallingForm,
            request: List[Mapping[str, str]]) -> FullRequest:
        """Validates and transforms the "request" block from a sample config.

           In the initial request, each dict has a "field" key that maps to a dotted
           variable name, e.g. clam.shell.

           The only required keys in each dict are "field" and value".
           Optional keys are "input_parameter", "value_is_file". and "comment".
           All values in the initial request are strings except for the value
           for "value_is_file", which is a bool.

           The TransformedRequest structure of the return value has four fields:
           "base", "body", "single", and "pattern",
           where "base" maps to the top level attribute name,
           "body" maps to a list of subfield assignment definitions, "single"
           maps to a singleton attribute assignment structure with no "field" value,
           and "pattern" is a resource name pattern string if the request describes
           resource name construction.
           The "field" attribute in the requests in a "body" list have their prefix stripped;
           the request in a "single" attribute has no "field" attribute.

           Note: gRPC API methods only take one parameter (ignoring client-side streaming).
                 The reason that GAPIC client library API methods may take multiple parameters
                 is a workaround to provide idiomatic protobuf support within python.
                 The different 'bases' are really attributes for the singular request parameter.

           TODO: properly handle subfields, indexing, and so forth.
           TODO: Add/transform to list repeated element fields.
                 Requires proto/method/message descriptors.

           E.g. [{"field": "clam.shell", "value": "10 kg", "input_parameter": "shell"},
                 {"field": "clam.pearls", "value": "3"},
                 {"field": "squid.mantle", "value": "100 kg"},
                 {"field": "whelk", "value": "speckled"}]
                  ->
                [TransformedRequest(
                     base="clam",
                     body=[AttributeRequestSetup(field="shell",
                                                 value="10 kg",
                                                 input_parameter="shell"),
                           AttributeRequestSetup(field="pearls", value="3")],
                     single=None),
                 TransformedRequest(base="squid",
                                    body=[AttributeRequestSetup(field="mantle",
                                                                value="100 kg")],
                                    single=None),
                 TransformedRequest(base="whelk",
                                    body=None,
                                    single=AttributeRequestSetup(value="speckled))]

           The transformation makes it easier to set up request parameters in jinja
           because it doesn't have to engage in prefix detection, validation,
           or aggregation logic.


        Args:
            request (list[dict{str:str}]): The request body from the sample config

        Returns:
            List[TransformedRequest]: The transformed request block.

        Raises:
            InvalidRequestSetup: If a dict in the request lacks a "field" key,
                                 a "value" key, if there is an unexpected keyword,
                                 or if more than one base parameter is given for
                                 a client-side streaming calling form.
            BadAttributeLookup: If a request field refers to a non-existent field
                                in the request message type.
            ResourceRequestMismatch: If a request attempts to describe both
                                     attribute manipulation and resource name
                                     construction.

        """
        base_param_to_attrs: Dict[str,
                                  RequestEntry] = defaultdict(RequestEntry)
        for r in request:
            r_dup = dict(r)
            val = r_dup.get("value")
            if not val:
                raise types.InvalidRequestSetup(
                    "Missing keyword in request entry: 'value'")

            field = r_dup.get("field")
            if not field:
                raise types.InvalidRequestSetup(
                    "Missing keyword in request entry: 'field'")

            spurious_kwords = set(r_dup.keys()) - self.VALID_REQUEST_KWORDS
            if spurious_kwords:
                raise types.InvalidRequestSetup(
                    "Spurious keyword(s) in request entry: {}".format(
                        ", ".join(f"'{kword}'" for kword in spurious_kwords)))

            input_parameter = r_dup.get("input_parameter")
            if input_parameter:
                self._handle_lvalue(
                    input_parameter,
                    wrappers.Field(
                        field_pb=descriptor_pb2.FieldDescriptorProto()),
                )

            # The percentage sign is used for setting up resource based requests
            percent_idx = field.find("%")
            if percent_idx == -1:
                base_param, attr = self._normal_request_setup(
                    base_param_to_attrs, val, r_dup, field)

                request_entry = base_param_to_attrs.get(base_param)
                if request_entry and request_entry.is_resource_request:
                    raise types.ResourceRequestMismatch(
                        f"Request setup mismatch for base: {base_param}")

                base_param_to_attrs[base_param].attrs.append(attr)
            else:
                # It's a resource based request.
                base_param, resource_attr = (
                    field[:percent_idx],
                    field[percent_idx + 1:],
                )
                request_entry = base_param_to_attrs.get(base_param)
                if request_entry and not request_entry.is_resource_request:
                    raise types.ResourceRequestMismatch(
                        f"Request setup mismatch for base: {base_param}")

                if not self.request_type_.fields.get(base_param):
                    raise types.BadAttributeLookup(
                        "Method request type {} has no attribute: '{}'".format(
                            self.request_type_, base_param))

                r_dup["field"] = resource_attr
                request_entry = base_param_to_attrs[base_param]
                request_entry.is_resource_request = True
                request_entry.attrs.append(
                    AttributeRequestSetup(**r_dup)  # type: ignore
                )

        client_streaming_forms = {
            types.CallingForm.RequestStreamingClient,
            types.CallingForm.RequestStreamingBidi,
        }

        if len(base_param_to_attrs
               ) > 1 and calling_form in client_streaming_forms:
            raise types.InvalidRequestSetup(
                "Too many base parameters for client side streaming form")

        # We can only flatten a collection of request parameters if they're a
        # subset of the flattened fields of the method.
        flattenable = self.flattenable_fields >= set(base_param_to_attrs)
        return FullRequest(
            request_list=[
                TransformedRequest.build(
                    self.request_type_,
                    self.api_schema_,
                    key,
                    val.attrs,
                    val.is_resource_request,
                ) for key, val in base_param_to_attrs.items()
            ],
            flattenable=False,
        )
    def validate_and_transform_request(
            self, calling_form: types.CallingForm,
            request: List[Mapping[str, str]]) -> List[TransformedRequest]:
        """Validates and transforms the "request" block from a sample config.

           In the initial request, each dict has a "field" key that maps to a dotted
           variable name, e.g. clam.shell.

           The only required keys in each dict are "field" and value".
           Optional keys are "input_parameter", "value_is_file". and "comment".
           All values in the initial request are strings except for the value
           for "value_is_file", which is a bool.

           The TransformedRequest structure of the return value has three fields:
           "base", "body", and "single", where "base" maps to the top level attribute name,
           "body" maps to a list of subfield assignment definitions, and "single"
           maps to a singleton attribute assignment structure with no "field" value.
           The "field" attribute in the requests in a "body" list have their prefix stripped;
           the request in a "single" attribute has no "field" attribute.

           Note: gRPC API methods only take one parameter (ignoring client-side streaming).
                 The reason that GAPIC client library API methods may take multiple parameters
                 is a workaround to provide idiomatic protobuf support within python.
                 The different 'bases' are really attributes for the singular request parameter.

           TODO: properly handle subfields, indexing, and so forth.
           TODO: Add/transform to list repeated element fields.
                 Requires proto/method/message descriptors.

           E.g. [{"field": "clam.shell", "value": "10 kg", "input_parameter": "shell"},
                 {"field": "clam.pearls", "value": "3"},
                 {"field": "squid.mantle", "value": "100 kg"},
                 {"field": "whelk", "value": "speckled"}]
                  ->
                [TransformedRequest(
                     base="clam",
                     body=[AttributeRequestSetup(field="shell",
                                                 value="10 kg",
                                                 input_parameter="shell"),
                           AttributeRequestSetup(field="pearls", value="3")],
                     single=None),
                 TransformedRequest(base="squid",
                                    body=[AttributeRequestSetup(field="mantle",
                                                                value="100 kg")],
                                    single=None),
                 TransformedRequest(base="whelk",
                                    body=None,
                                    single=AttributeRequestSetup(value="speckled))]

           The transformation makes it easier to set up request parameters in jinja
           because it doesn't have to engage in prefix detection, validation,
           or aggregation logic.


        Args:
            request (list[dict{str:str}]): The request body from the sample config

        Returns:
            List[TransformedRequest]: The transformed request block.

        Raises:
            InvalidRequestSetup: If a dict in the request lacks a "field" key,
                                 a "value" key, if there is an unexpected keyword,
                                 or if more than one base parameter is given for
                                 a client-side streaming calling form.
            BadAttributeLookup: If a request field refers to a non-existent field
                                in the request message type.

        """
        base_param_to_attrs: Dict[
            str, List[AttributeRequestSetup]] = defaultdict(list)

        for r in request:
            duplicate = dict(r)
            val = duplicate.get("value")
            if not val:
                raise types.InvalidRequestSetup(
                    "Missing keyword in request entry: 'value'")

            field = duplicate.get("field")
            if not field:
                raise types.InvalidRequestSetup(
                    "Missing keyword in request entry: 'field'")

            spurious_keywords = set(duplicate.keys()) - {
                "value", "field", "value_is_file", "input_parameter", "comment"
            }
            if spurious_keywords:
                raise types.InvalidRequestSetup(
                    "Spurious keyword(s) in request entry: {}".format(
                        ", ".join(f"'{kword}'"
                                  for kword in spurious_keywords)))

            input_parameter = duplicate.get("input_parameter")
            if input_parameter:
                self._handle_lvalue(
                    input_parameter,
                    wrappers.Field(
                        field_pb=descriptor_pb2.FieldDescriptorProto()))

            attr_chain = field.split(".")
            base = self.request_type_
            for i, attr_name in enumerate(attr_chain):
                attr = base.fields.get(attr_name)
                if not attr:
                    raise types.BadAttributeLookup(
                        "Method request type {} has no attribute: '{}'".format(
                            self.request_type_.type, attr_name))

                if attr.message:
                    base = attr.message
                elif attr.enum:
                    # A little bit hacky, but 'values' is a list, and this is the easiest
                    # way to verify that the value is a valid enum variant.
                    witness = any(e.name == val for e in attr.enum.values)
                    if not witness:
                        raise types.InvalidEnumVariant(
                            "Invalid variant for enum {}: '{}'".format(
                                attr, val))
                    # Python code can set protobuf enums from strings.
                    # This is preferable to adding the necessary import statement
                    # and requires less munging of the assigned value
                    duplicate["value"] = f"'{val}'"
                    break
                else:
                    raise TypeError

            if i != len(attr_chain) - 1:
                # We broke out of the loop after processing an enum.
                extra_attrs = ".".join(attr_chain[i:])
                raise types.InvalidEnumVariant(
                    f"Attempted to reference attributes of enum value: '{extra_attrs}'"
                )

            if len(attr_chain) > 1:
                duplicate["field"] = ".".join(attr_chain[1:])
            else:
                # Because of the way top level attrs get rendered,
                # there can't be duplicates.
                # This is admittedly a bit of a hack.
                if attr_chain[0] in base_param_to_attrs:
                    raise types.InvalidRequestSetup(
                        "Duplicated top level field in request block: '{}'".
                        format(attr_chain[0]))
                del duplicate["field"]

            # Mypy isn't smart enough to handle dictionary unpacking,
            # so disable it for the AttributeRequestSetup ctor call.
            base_param_to_attrs[attr_chain[0]].append(
                AttributeRequestSetup(**duplicate))  # type: ignore

        client_streaming_forms = {
            types.CallingForm.RequestStreamingClient,
            types.CallingForm.RequestStreamingBidi,
        }

        if len(base_param_to_attrs
               ) > 1 and calling_form in client_streaming_forms:
            raise types.InvalidRequestSetup(
                "Too many base parameters for client side streaming form")

        return [(TransformedRequest(base=key, body=val, single=None)
                 if val[0].field else TransformedRequest(
                     base=key, body=None, single=val[0]))
                for key, val in base_param_to_attrs.items()]