def validate_recursively(expression, scope, depth=0):
first_dot = expression.find(".")
base = expression[:first_dot] if first_dot > 0 else expression
match = self.EXPRESSION_ATTR_RE.match(base)
if not match:
raise types.BadAttributeLookup(
f"Badly formed attribute expression: {expression}")
name, idxed, mapped = (
match.groupdict()["attr_name"],
bool(match.groupdict()["index"]),
bool(match.groupdict()["key"]),
)
field = scope.get(name)
if not field:
exception_class = (types.BadAttributeLookup if depth else
types.UndefinedVariableReference)
raise exception_class(f"No such variable or attribute: {name}")
# Invalid input
if (idxed or mapped) and not field.repeated:
raise types.BadAttributeLookup(
f"Collection lookup on non-repeated field: {base}")
# Can only ignore indexing or mapping in an indexed (or mapped) field
# if it is the terminal point in the expression.
if field.repeated and not (idxed or mapped) and first_dot != -1:
raise types.BadAttributeLookup(
("Accessing attribute on a non-terminal collection without"
f"indexing into the collection: {base}"))
message = field.message
scope = dict(message.fields) if message else {}
# Can only map message types, not enums
if mapped:
# See https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/descriptor.proto#L496
# for a better understanding of how map attributes are handled in protobuf
if not message or not message.options.map_field:
raise types.BadAttributeLookup(
f"Badly formed mapped field: {base}")
value_field = message.fields.get("value")
if not value_field:
raise types.BadAttributeLookup(
f"Mapped attribute has no value field: {base}")
value_message = value_field.message
if not value_message:
raise types.BadAttributeLookup(
f"Mapped value field is not a message: {base}")
if first_dot != -1:
scope = value_message.fields
# Terminus of the expression.
if first_dot == -1:
return field
# Enums and primitives are only allowed at the tail of an expression.
if not message:
raise types.BadAttributeLookup(
f"Non-terminal attribute is not a message: {base}")
return validate_recursively(expression[first_dot + 1:], scope,
depth + 1)
def _normal_request_setup(self, base_param_to_attrs, val, request, field):
"""validates and transforms non-resource-based request entries.
Private method, lifted out to make validate_and_transform_request cleaner.
Args:
base_param_to_attrs ({str:RequestEntry}):
val (str): The value to which the terminal field will be set
(only used if the terminus is an enum)
request (str:str): The request dictionary read in from the config.
field (str): The value of the "field" parameter in the request entry.
Returns:
Tuple[str, AttributeRequestSetup]
"""
base = self.request_type_
attr_chain = field.split(".")
for i, attr_name in enumerate(attr_chain):
attr = base.fields.get(attr_name)
if not attr:
raise types.BadAttributeLookup(
"Method request type {} has no attribute: '{}'".format(
self.request_type_, attr_name))
if attr.message:
base = attr.message
elif attr.enum:
# A little bit hacky, but 'values' is a list, and this is the easiest
# way to verify that the value is a valid enum variant.
witness = any(e.name == val for e in attr.enum.values)
if not witness:
raise types.InvalidEnumVariant(
"Invalid variant for enum {}: '{}'".format(attr, val))
break
elif attr.is_primitive:
# Only valid if this is the last attribute in the chain.
break
else:
raise TypeError(
f"Could not handle attribute '{attr_name}' of type: {attr.type}"
)
if i != len(attr_chain) - 1:
# We broke out of the loop after processing an enum or a primitive.
extra_attrs = ".".join(attr_chain[i:])
raise types.NonTerminalPrimitiveOrEnum(
f"Attempted to reference attributes of enum value or primitive type: '{extra_attrs}'"
)
if len(attr_chain) > 1:
request["field"] = ".".join(attr_chain[1:])
else:
# Because of the way top level attrs get rendered,
# there can't be duplicates.
# This is admittedly a bit of a hack.
if attr_chain[0] in base_param_to_attrs:
raise types.InvalidRequestSetup(
"Duplicated top level field in request block: '{}'".format(
attr_chain[0]))
del request["field"]
if isinstance(request["value"], str):
# Passing value through json is a safe and simple way of
# making sure strings are properly wrapped and quotes escaped.
# This statement both wraps enums in quotes and escapes quotes
# in string values passed as parameters.
#
# Python code can set protobuf enums from strings.
# This is preferable to adding the necessary import statement
# and requires less munging of the assigned value
request["value"] = json.dumps(request["value"])
# Mypy isn't smart enough to handle dictionary unpacking,
# so disable it for the AttributeRequestSetup ctor call.
return attr_chain[0], AttributeRequestSetup(**request) # type: ignore
def validate_and_transform_request(
self, calling_form: types.CallingForm,
request: List[Mapping[str, str]]) -> FullRequest:
"""Validates and transforms the "request" block from a sample config.
In the initial request, each dict has a "field" key that maps to a dotted
variable name, e.g. clam.shell.
The only required keys in each dict are "field" and value".
Optional keys are "input_parameter", "value_is_file". and "comment".
All values in the initial request are strings except for the value
for "value_is_file", which is a bool.
The TransformedRequest structure of the return value has four fields:
"base", "body", "single", and "pattern",
where "base" maps to the top level attribute name,
"body" maps to a list of subfield assignment definitions, "single"
maps to a singleton attribute assignment structure with no "field" value,
and "pattern" is a resource name pattern string if the request describes
resource name construction.
The "field" attribute in the requests in a "body" list have their prefix stripped;
the request in a "single" attribute has no "field" attribute.
Note: gRPC API methods only take one parameter (ignoring client-side streaming).
The reason that GAPIC client library API methods may take multiple parameters
is a workaround to provide idiomatic protobuf support within python.
The different 'bases' are really attributes for the singular request parameter.
TODO: properly handle subfields, indexing, and so forth.
TODO: Add/transform to list repeated element fields.
Requires proto/method/message descriptors.
E.g. [{"field": "clam.shell", "value": "10 kg", "input_parameter": "shell"},
{"field": "clam.pearls", "value": "3"},
{"field": "squid.mantle", "value": "100 kg"},
{"field": "whelk", "value": "speckled"}]
->
[TransformedRequest(
base="clam",
body=[AttributeRequestSetup(field="shell",
value="10 kg",
input_parameter="shell"),
AttributeRequestSetup(field="pearls", value="3")],
single=None),
TransformedRequest(base="squid",
body=[AttributeRequestSetup(field="mantle",
value="100 kg")],
single=None),
TransformedRequest(base="whelk",
body=None,
single=AttributeRequestSetup(value="speckled))]
The transformation makes it easier to set up request parameters in jinja
because it doesn't have to engage in prefix detection, validation,
or aggregation logic.
Args:
request (list[dict{str:str}]): The request body from the sample config
Returns:
List[TransformedRequest]: The transformed request block.
Raises:
InvalidRequestSetup: If a dict in the request lacks a "field" key,
a "value" key, if there is an unexpected keyword,
or if more than one base parameter is given for
a client-side streaming calling form.
BadAttributeLookup: If a request field refers to a non-existent field
in the request message type.
ResourceRequestMismatch: If a request attempts to describe both
attribute manipulation and resource name
construction.
"""
base_param_to_attrs: Dict[str,
RequestEntry] = defaultdict(RequestEntry)
for r in request:
r_dup = dict(r)
val = r_dup.get("value")
if not val:
raise types.InvalidRequestSetup(
"Missing keyword in request entry: 'value'")
field = r_dup.get("field")
if not field:
raise types.InvalidRequestSetup(
"Missing keyword in request entry: 'field'")
spurious_kwords = set(r_dup.keys()) - self.VALID_REQUEST_KWORDS
if spurious_kwords:
raise types.InvalidRequestSetup(
"Spurious keyword(s) in request entry: {}".format(
", ".join(f"'{kword}'" for kword in spurious_kwords)))
input_parameter = r_dup.get("input_parameter")
if input_parameter:
self._handle_lvalue(
input_parameter,
wrappers.Field(
field_pb=descriptor_pb2.FieldDescriptorProto()),
)
# The percentage sign is used for setting up resource based requests
percent_idx = field.find("%")
if percent_idx == -1:
base_param, attr = self._normal_request_setup(
base_param_to_attrs, val, r_dup, field)
request_entry = base_param_to_attrs.get(base_param)
if request_entry and request_entry.is_resource_request:
raise types.ResourceRequestMismatch(
f"Request setup mismatch for base: {base_param}")
base_param_to_attrs[base_param].attrs.append(attr)
else:
# It's a resource based request.
base_param, resource_attr = (
field[:percent_idx],
field[percent_idx + 1:],
)
request_entry = base_param_to_attrs.get(base_param)
if request_entry and not request_entry.is_resource_request:
raise types.ResourceRequestMismatch(
f"Request setup mismatch for base: {base_param}")
if not self.request_type_.fields.get(base_param):
raise types.BadAttributeLookup(
"Method request type {} has no attribute: '{}'".format(
self.request_type_, base_param))
r_dup["field"] = resource_attr
request_entry = base_param_to_attrs[base_param]
request_entry.is_resource_request = True
request_entry.attrs.append(
AttributeRequestSetup(**r_dup) # type: ignore
)
client_streaming_forms = {
types.CallingForm.RequestStreamingClient,
types.CallingForm.RequestStreamingBidi,
}
if len(base_param_to_attrs
) > 1 and calling_form in client_streaming_forms:
raise types.InvalidRequestSetup(
"Too many base parameters for client side streaming form")
# We can only flatten a collection of request parameters if they're a
# subset of the flattened fields of the method.
flattenable = self.flattenable_fields >= set(base_param_to_attrs)
return FullRequest(
request_list=[
TransformedRequest.build(
self.request_type_,
self.api_schema_,
key,
val.attrs,
val.is_resource_request,
) for key, val in base_param_to_attrs.items()
],
flattenable=False,
)
def validate_and_transform_request(
self, calling_form: types.CallingForm,
request: List[Mapping[str, str]]) -> List[TransformedRequest]:
"""Validates and transforms the "request" block from a sample config.
In the initial request, each dict has a "field" key that maps to a dotted
variable name, e.g. clam.shell.
The only required keys in each dict are "field" and value".
Optional keys are "input_parameter", "value_is_file". and "comment".
All values in the initial request are strings except for the value
for "value_is_file", which is a bool.
The TransformedRequest structure of the return value has three fields:
"base", "body", and "single", where "base" maps to the top level attribute name,
"body" maps to a list of subfield assignment definitions, and "single"
maps to a singleton attribute assignment structure with no "field" value.
The "field" attribute in the requests in a "body" list have their prefix stripped;
the request in a "single" attribute has no "field" attribute.
Note: gRPC API methods only take one parameter (ignoring client-side streaming).
The reason that GAPIC client library API methods may take multiple parameters
is a workaround to provide idiomatic protobuf support within python.
The different 'bases' are really attributes for the singular request parameter.
TODO: properly handle subfields, indexing, and so forth.
TODO: Add/transform to list repeated element fields.
Requires proto/method/message descriptors.
E.g. [{"field": "clam.shell", "value": "10 kg", "input_parameter": "shell"},
{"field": "clam.pearls", "value": "3"},
{"field": "squid.mantle", "value": "100 kg"},
{"field": "whelk", "value": "speckled"}]
->
[TransformedRequest(
base="clam",
body=[AttributeRequestSetup(field="shell",
value="10 kg",
input_parameter="shell"),
AttributeRequestSetup(field="pearls", value="3")],
single=None),
TransformedRequest(base="squid",
body=[AttributeRequestSetup(field="mantle",
value="100 kg")],
single=None),
TransformedRequest(base="whelk",
body=None,
single=AttributeRequestSetup(value="speckled))]
The transformation makes it easier to set up request parameters in jinja
because it doesn't have to engage in prefix detection, validation,
or aggregation logic.
Args:
request (list[dict{str:str}]): The request body from the sample config
Returns:
List[TransformedRequest]: The transformed request block.
Raises:
InvalidRequestSetup: If a dict in the request lacks a "field" key,
a "value" key, if there is an unexpected keyword,
or if more than one base parameter is given for
a client-side streaming calling form.
BadAttributeLookup: If a request field refers to a non-existent field
in the request message type.
"""
base_param_to_attrs: Dict[
str, List[AttributeRequestSetup]] = defaultdict(list)
for r in request:
duplicate = dict(r)
val = duplicate.get("value")
if not val:
raise types.InvalidRequestSetup(
"Missing keyword in request entry: 'value'")
field = duplicate.get("field")
if not field:
raise types.InvalidRequestSetup(
"Missing keyword in request entry: 'field'")
spurious_keywords = set(duplicate.keys()) - {
"value", "field", "value_is_file", "input_parameter", "comment"
}
if spurious_keywords:
raise types.InvalidRequestSetup(
"Spurious keyword(s) in request entry: {}".format(
", ".join(f"'{kword}'"
for kword in spurious_keywords)))
input_parameter = duplicate.get("input_parameter")
if input_parameter:
self._handle_lvalue(
input_parameter,
wrappers.Field(
field_pb=descriptor_pb2.FieldDescriptorProto()))
attr_chain = field.split(".")
base = self.request_type_
for i, attr_name in enumerate(attr_chain):
attr = base.fields.get(attr_name)
if not attr:
raise types.BadAttributeLookup(
"Method request type {} has no attribute: '{}'".format(
self.request_type_.type, attr_name))
if attr.message:
base = attr.message
elif attr.enum:
# A little bit hacky, but 'values' is a list, and this is the easiest
# way to verify that the value is a valid enum variant.
witness = any(e.name == val for e in attr.enum.values)
if not witness:
raise types.InvalidEnumVariant(
"Invalid variant for enum {}: '{}'".format(
attr, val))
# Python code can set protobuf enums from strings.
# This is preferable to adding the necessary import statement
# and requires less munging of the assigned value
duplicate["value"] = f"'{val}'"
break
else:
raise TypeError
if i != len(attr_chain) - 1:
# We broke out of the loop after processing an enum.
extra_attrs = ".".join(attr_chain[i:])
raise types.InvalidEnumVariant(
f"Attempted to reference attributes of enum value: '{extra_attrs}'"
)
if len(attr_chain) > 1:
duplicate["field"] = ".".join(attr_chain[1:])
else:
# Because of the way top level attrs get rendered,
# there can't be duplicates.
# This is admittedly a bit of a hack.
if attr_chain[0] in base_param_to_attrs:
raise types.InvalidRequestSetup(
"Duplicated top level field in request block: '{}'".
format(attr_chain[0]))
del duplicate["field"]
# Mypy isn't smart enough to handle dictionary unpacking,
# so disable it for the AttributeRequestSetup ctor call.
base_param_to_attrs[attr_chain[0]].append(
AttributeRequestSetup(**duplicate)) # type: ignore
client_streaming_forms = {
types.CallingForm.RequestStreamingClient,
types.CallingForm.RequestStreamingBidi,
}
if len(base_param_to_attrs
) > 1 and calling_form in client_streaming_forms:
raise types.InvalidRequestSetup(
"Too many base parameters for client side streaming form")
return [(TransformedRequest(base=key, body=val, single=None)
if val[0].field else TransformedRequest(
base=key, body=None, single=val[0]))
for key, val in base_param_to_attrs.items()]