def encode_union(self, validator, value):
if value._tag is None:
raise bv.ValidationError('no tag set')
if not validator.definition._is_tag_present(value._tag,
self.caller_permissions):
raise bv.ValidationError(
"caller does not have access to '{}' tag".format(value._tag))
field_validator = validator.definition._get_val_data_type(
value._tag, self.caller_permissions)
is_none = isinstance(field_validator, bv.Void) \
or (isinstance(field_validator, bv.Nullable)
and value._value is None)
def encode_sub(sub_validator, sub_value, parent_tag):
try:
encoded_val = self.encode_sub(sub_validator, sub_value)
except bv.ValidationError as exc:
exc.add_parent(parent_tag)
raise
else:
return encoded_val
if self.old_style:
if field_validator is None:
return value._tag
elif is_none:
return value._tag
else:
encoded_val = encode_sub(field_validator, value._value,
value._tag)
return {value._tag: encoded_val}
elif is_none:
return {'.tag': value._tag}
else:
encoded_val = encode_sub(field_validator, value._value, value._tag)
if isinstance(field_validator, bv.Nullable):
# We've already checked for the null case above,
# so now we're only interested in what the
# wrapped validator is
field_validator = field_validator.validator
if isinstance(field_validator, bv.Struct) \
and not isinstance(field_validator, bv.StructTree):
d = collections.OrderedDict(
) # type: typing.Dict[str, typing.Any]
d['.tag'] = value._tag
d.update(encoded_val)
return d
else:
return collections.OrderedDict((
('.tag', value._tag),
(value._tag, encoded_val),
))
def decode_struct(self, data_type, obj):
"""
The data_type argument must be a Struct.
See json_compat_obj_decode() for argument descriptions.
"""
if obj is None and data_type.has_default():
return data_type.get_default()
elif not isinstance(obj, dict):
raise bv.ValidationError('expected object, got %s' %
bv.generic_type_name(obj))
all_fields = data_type.definition._all_fields_
for extra_permission in self.caller_permissions.permissions:
all_extra_fields = '_all_{}_fields_'.format(extra_permission)
all_fields = all_fields + getattr(data_type.definition,
all_extra_fields, [])
if self.strict:
all_field_names = data_type.definition._all_field_names_
for extra_permission in self.caller_permissions.permissions:
all_extra_field_names = '_all_{}_field_names_'.format(
extra_permission)
all_field_names = all_field_names.union(
getattr(data_type.definition, all_extra_field_names, {}))
for key in obj:
if (key not in all_field_names and not key.startswith('.tag')):
raise bv.ValidationError("unknown field '%s'" % key)
ins = data_type.definition()
self.decode_struct_fields(ins, all_fields, obj)
# Check that all required fields have been set.
data_type.validate_fields_only_with_permissions(
ins, self.caller_permissions)
return ins
def _encode_union_old(data_type, obj, alias_validators, for_msgpack):
"""
The data_type argument must be a Union.
See json_encode() for argument descriptions.
"""
if obj._tag is None:
raise bv.ValidationError('no tag set')
field_data_type = data_type.definition._tagmap[obj._tag]
if field_data_type is None:
return obj._tag
else:
if (isinstance(field_data_type, bv.Void) or
(isinstance(field_data_type, bv.Nullable) and
obj._value is None)):
return obj._tag
else:
try:
encoded_val = _json_compat_obj_encode_helper(
field_data_type, obj._value, alias_validators, True,
for_msgpack)
except bv.ValidationError as e:
e.add_parent(obj._tag)
raise
else:
return {obj._tag: encoded_val}
def _encode_union(data_type, obj, alias_validators, for_msgpack):
"""
The data_type argument must be a Union.
See json_encode() for argument descriptions.
"""
if obj._tag is None:
raise bv.ValidationError('no tag set')
field_data_type = data_type.definition._tagmap[obj._tag]
if (isinstance(field_data_type, bv.Void) or
(isinstance(field_data_type, bv.Nullable) and obj._value is None)):
return {'.tag': obj._tag}
else:
try:
encoded_val = _json_compat_obj_encode_helper(
field_data_type, obj._value, alias_validators, False,
for_msgpack)
except bv.ValidationError as e:
e.add_parent(obj._tag)
raise
else:
if isinstance(field_data_type, bv.Nullable):
# We've already checked for the null case above, so now we're
# only interested in what the wrapped validator is.
field_data_type = field_data_type.validator
if (isinstance(field_data_type, bv.Struct) and
not isinstance(field_data_type, bv.StructTree)):
d = collections.OrderedDict()
d['.tag'] = obj._tag
d.update(encoded_val)
return d
else:
return collections.OrderedDict([
('.tag', obj._tag),
(obj._tag, encoded_val)])
def _encode_struct(data_type, obj, alias_validators, old_style, for_msgpack):
"""
The data_type argument must be a Struct or StructTree.
See json_encode() for argument descriptions.
"""
# We skip validation of fields with primitive data types in structs and
# unions because they've already been validated on assignment.
d = collections.OrderedDict()
for field_name, field_data_type in data_type.definition._all_fields_:
try:
val = getattr(obj, field_name)
except AttributeError as e:
raise bv.ValidationError(e.args[0])
presence_key = '_%s_present' % field_name
if val is not None and getattr(obj, presence_key):
# This check makes sure that we don't serialize absent struct
# fields as null, even if there is a default.
try:
d[field_name] = _json_compat_obj_encode_helper(
field_data_type, val, alias_validators, old_style,
for_msgpack)
except bv.ValidationError as e:
e.add_parent(field_name)
raise
return d
def encode_struct(self, validator, value):
# Skip validation of fields with primitive data types because
# they've already been validated on assignment
d = collections.OrderedDict() # type: typing.Dict[str, typing.Any]
all_fields = validator.definition._all_fields_
for extra_permission in self.caller_permissions.permissions:
all_fields_name = '_all_{}_fields_'.format(extra_permission)
all_fields = all_fields + getattr(validator.definition,
all_fields_name, [])
for field_name, field_validator in all_fields:
try:
field_value = getattr(value, field_name)
except AttributeError as exc:
raise bv.ValidationError(exc.args[0])
presence_key = '_%s_present' % field_name
if field_value is not None \
and getattr(value, presence_key):
# Only serialize struct fields that have been explicitly
# set, even if there is a default
try:
d[field_name] = self.encode_sub(field_validator,
field_value)
except bv.ValidationError as exc:
exc.add_parent(field_name)
raise
return d
def encode_sub(self, validator, value):
# type: (bv.Validator, typing.Any) -> typing.Any
"""
Callback intended to be called by other ``encode`` methods to
delegate encoding of sub-values. Arguments have the same semantics
as with the ``encode`` method.
"""
if isinstance(validator, bv.List):
# Because Lists are mutable, we always validate them during
# serialization
validate_f = validator.validate # type: typing.Callable[[typing.Any], None]
encode_f = self.encode_list # type: typing.Callable[[typing.Any, typing.Any], typing.Any] # noqa: E501
elif isinstance(validator, bv.Map):
# Also validate maps during serialization because they are also mutable
validate_f = validator.validate
encode_f = self.encode_map
elif isinstance(validator, bv.Nullable):
validate_f = validator.validate
encode_f = self.encode_nullable
elif isinstance(validator, bv.Primitive):
validate_f = validator.validate
encode_f = self.encode_primitive
elif isinstance(validator, bv.Struct):
if isinstance(validator, bv.StructTree):
if self.caller_permissions.permissions:
def validate_with_permissions(val):
validator.validate_with_permissions(
val, self.caller_permissions)
validate_f = validate_with_permissions
else:
validate_f = validator.validate
encode_f = self.encode_struct_tree
else:
# Fields are already validated on assignment
if self.caller_permissions.permissions:
def validate_with_permissions(val):
validator.validate_with_permissions(
val, self.caller_permissions)
validate_f = validate_with_permissions
else:
validate_f = validator.validate_type_only
encode_f = self.encode_struct
elif isinstance(validator, bv.Union):
# Fields are already validated on assignment
validate_f = validator.validate_type_only
encode_f = self.encode_union
else:
raise bv.ValidationError('Unsupported data type {}'.format(
type(validator).__name__))
validate_f(value)
return encode_f(validator, value)
def make_stone_friendly(self, data_type, val, validate):
"""
Convert a Python object to a type that will pass validation by its
validator.
Validation by ``alias_validators`` is performed even if ``validate`` is
false.
"""
if isinstance(data_type, bv.Timestamp):
try:
ret = datetime.datetime.strptime(val, data_type.format)
except:
# datetime.datetime.strptime(val, data_type.format) returned NoneType. Trying alterntive
pass
try:
ret = datetime.datetime(
*(time.strptime(val, data_type.format)[0:6]))
except (TypeError, ValueError) as e:
raise bv.ValidationError(e.args[0])
elif isinstance(data_type, bv.Bytes):
if self.for_msgpack:
if isinstance(val, six.text_type):
ret = val.encode('utf-8')
else:
ret = val
else:
try:
ret = base64.b64decode(val)
except TypeError:
raise bv.ValidationError('invalid base64-encoded bytes')
elif isinstance(data_type, bv.Void):
if self.strict and val is not None:
raise bv.ValidationError("expected null, got value")
return None
else:
if validate:
if self.caller_permissions.permissions:
data_type.validate_with_permissions(
val, self.caller_permissions)
else:
data_type.validate(val)
ret = val
if self.alias_validators is not None and data_type in self.alias_validators:
self.alias_validators[data_type](ret)
return ret
def _make_stone_friendly(data_type, val, alias_validators, strict, validate,
for_msgpack):
"""
Convert a Python object to a type that will pass validation by its
validator.
Validation by ``alias_validators`` is performed even if ``validate`` is
false.
fix found at:
https://www.dropboxforum.com/t5/API-support/Upload-Error-with-v2-migration-from-v1/td-p/244561
"""
if isinstance(data_type, bv.Timestamp):
try:
ret = datetime.datetime.strptime(val, data_type.format)
except:
#print("datetime.datetime.strptime(val, data_type.format) returned NoneType. Trying alterntive")
pass
try:
ret = datetime.datetime(
*(time.strptime(val, data_type.format)[0:6]))
except (TypeError, ValueError) as e:
raise bv.ValidationError(e.args[0])
elif isinstance(data_type, bv.Bytes):
if for_msgpack:
if isinstance(val, six.text_type):
ret = val.encode('utf-8')
else:
ret = val
else:
try:
ret = base64.b64decode(val)
except TypeError:
raise bv.ValidationError('invalid base64-encoded bytes')
elif isinstance(data_type, bv.Void):
if strict and val is not None:
raise bv.ValidationError("expected null, got value")
return None
else:
if validate:
data_type.validate(val)
ret = val
if alias_validators is not None and data_type in alias_validators:
alias_validators[data_type](ret)
return ret
def decode_list(self, data_type, obj):
"""
The data_type argument must be a List.
See json_compat_obj_decode() for argument descriptions.
"""
if not isinstance(obj, list):
raise bv.ValidationError(
'expected list, got %s' % bv.generic_type_name(obj))
return [
self.json_compat_obj_decode_helper(data_type.item_validator, item)
for item in obj]
def decode_map(self, data_type, obj):
"""
The data_type argument must be a Map.
See json_compat_obj_decode() for argument descriptions.
"""
if not isinstance(obj, dict):
raise bv.ValidationError(
'expected dict, got %s' % bv.generic_type_name(obj))
return {
self.json_compat_obj_decode_helper(data_type.key_validator, key):
self.json_compat_obj_decode_helper(data_type.value_validator, value)
for key, value in obj.items()
}
def _decode_struct(data_type, obj, alias_validators, strict, old_style,
for_msgpack):
"""
The data_type argument must be a Struct.
See json_compat_obj_decode() for argument descriptions.
"""
if obj is None and data_type.has_default():
return data_type.get_default()
elif not isinstance(obj, dict):
raise bv.ValidationError('expected object, got %s' %
bv.generic_type_name(obj))
if strict:
for key in obj:
if (key not in data_type.definition._all_field_names_
and not key.startswith('.tag')):
raise bv.ValidationError("unknown field '%s'" % key)
ins = data_type.definition()
_decode_struct_fields(ins, data_type.definition._all_fields_, obj,
alias_validators, strict, old_style, for_msgpack)
# Check that all required fields have been set.
data_type.validate_fields_only(ins)
return ins
def decode_union_old(self, data_type, obj):
"""
The data_type argument must be a Union.
See json_compat_obj_decode() for argument descriptions.
"""
val = None
if isinstance(obj, six.string_types):
# Union member has no associated value
tag = obj
if data_type.definition._is_tag_present(tag,
self.caller_permissions):
val_data_type = data_type.definition._get_val_data_type(
tag, self.caller_permissions)
if not isinstance(val_data_type, (bv.Void, bv.Nullable)):
raise bv.ValidationError(
"expected object for '%s', got symbol" % tag)
else:
if not self.strict and data_type.definition._catch_all:
tag = data_type.definition._catch_all
else:
raise bv.ValidationError("unknown tag '%s'" % tag)
elif isinstance(obj, dict):
# Union member has value
if len(obj) != 1:
raise bv.ValidationError('expected 1 key, got %s' % len(obj))
tag = list(obj)[0]
raw_val = obj[tag]
if data_type.definition._is_tag_present(tag,
self.caller_permissions):
val_data_type = data_type.definition._get_val_data_type(
tag, self.caller_permissions)
if isinstance(val_data_type, bv.Nullable) and raw_val is None:
val = None
elif isinstance(val_data_type, bv.Void):
if raw_val is None or not self.strict:
# If raw_val is None, then this is the more verbose
# representation of a void union member. If raw_val isn't
# None, then maybe the spec has changed, so check if we're
# in strict mode.
val = None
else:
raise bv.ValidationError('expected null, got %s' %
bv.generic_type_name(raw_val))
else:
try:
val = self.json_compat_obj_decode_helper(
val_data_type, raw_val)
except bv.ValidationError as e:
e.add_parent(tag)
raise
else:
if not self.strict and data_type.definition._catch_all:
tag = data_type.definition._catch_all
else:
raise bv.ValidationError("unknown tag '%s'" % tag)
else:
raise bv.ValidationError("expected string or object, got %s" %
bv.generic_type_name(obj))
return data_type.definition(tag, val)
def _make_stone_friendly(data_type, val, alias_validators, strict, validate,
for_msgpack):
"""
Convert a Python object to a type that will pass validation by its
validator.
Validation by ``alias_validators`` is performed even if ``validate`` is
false.
"""
if isinstance(data_type, bv.Timestamp):
try:
# modified for kodi's issues with datetime
ret = datetime.datetime(
*(time.strptime(val, data_type.format)[0:6]))
except (TypeError, ValueError) as e:
raise bv.ValidationError(e.args[0])
elif isinstance(data_type, bv.Bytes):
if for_msgpack:
if isinstance(val, six.text_type):
ret = val.encode('utf-8')
else:
ret = val
else:
try:
ret = base64.b64decode(val)
except TypeError:
raise bv.ValidationError('invalid base64-encoded bytes')
elif isinstance(data_type, bv.Void):
if strict and val is not None:
raise bv.ValidationError("expected null, got value")
return None
else:
if validate:
data_type.validate(val)
ret = val
if alias_validators is not None and data_type in alias_validators:
alias_validators[data_type](ret)
return ret
def _decode_list(
data_type, obj, alias_validators, strict, old_style, for_msgpack):
"""
The data_type argument must be a List.
See json_compat_obj_decode() for argument descriptions.
"""
if not isinstance(obj, list):
raise bv.ValidationError(
'expected list, got %s' % bv.generic_type_name(obj))
return [
_json_compat_obj_decode_helper(
data_type.item_validator, item, alias_validators, strict,
old_style, for_msgpack)
for item in obj]
def determine_struct_tree_subtype(self, data_type, obj):
"""
Searches through the JSON-object-compatible dict using the data type
definition to determine which of the enumerated subtypes `obj` is.
"""
if '.tag' not in obj:
raise bv.ValidationError("missing '.tag' key")
if not isinstance(obj['.tag'], six.string_types):
raise bv.ValidationError('expected string, got %s' %
bv.generic_type_name(obj['.tag']),
parent='.tag')
# Find the subtype the tags refer to
full_tags_tuple = (obj['.tag'], )
if full_tags_tuple in data_type.definition._tag_to_subtype_:
subtype = data_type.definition._tag_to_subtype_[full_tags_tuple]
if isinstance(subtype, bv.StructTree):
raise bv.ValidationError(
"tag '%s' refers to non-leaf subtype" %
('.'.join(full_tags_tuple)))
return subtype
else:
if self.strict:
# In strict mode, the entirety of the tag hierarchy should
# point to a known subtype.
raise bv.ValidationError("unknown subtype '%s'" %
'.'.join(full_tags_tuple))
else:
# If subtype was not found, use the base.
if data_type.definition._is_catch_all_:
return data_type
else:
raise bv.ValidationError(
"unknown subtype '%s' and '%s' is not a catch-all" %
('.'.join(full_tags_tuple),
data_type.definition.__name__))
def json_decode(data_type,
serialized_obj,
caller_permissions=None,
alias_validators=None,
strict=True,
old_style=False):
"""Performs the reverse operation of json_encode.
Args:
data_type (Validator): Validator for serialized_obj.
serialized_obj (str): The JSON string to deserialize.
caller_permissions (list): The list of raw-string caller permissions
with which to serialize.
alias_validators (Optional[Mapping[bv.Validator, Callable[[], None]]]):
Custom validation functions. These must raise bv.ValidationError on
failure.
strict (bool): If strict, then unknown struct fields will raise an
error, and unknown union variants will raise an error even if a
catch all field is specified. strict should only be used by a
recipient of serialized JSON if it's guaranteed that its Stone
specs are at least as recent as the senders it receives messages
from.
Returns:
The returned object depends on the input data_type.
- Boolean -> bool
- Bytes -> bytes
- Float -> float
- Integer -> long
- List -> list
- Map -> dict
- Nullable -> None or its wrapped type.
- String -> unicode (PY2) or str (PY3)
- Struct -> An instance of its definition attribute.
- Timestamp -> datetime.datetime
- Union -> An instance of its definition attribute.
"""
try:
deserialized_obj = json.loads(serialized_obj)
except ValueError:
raise bv.ValidationError('could not decode input as JSON')
else:
return json_compat_obj_decode(data_type,
deserialized_obj,
caller_permissions=caller_permissions,
alias_validators=alias_validators,
strict=strict,
old_style=old_style)
def _decode_map(data_type, obj, alias_validators, strict, old_style,
for_msgpack):
"""
The data_type argument must be a Map.
See json_compat_obj_decode() for argument descriptions.
"""
if not isinstance(obj, dict):
raise bv.ValidationError('expected dict, got %s' %
bv.generic_type_name(obj))
return {
_json_compat_obj_decode_helper(data_type.key_validator, key,
alias_validators, strict, old_style,
for_msgpack):
_json_compat_obj_decode_helper(data_type.value_validator, value,
alias_validators, strict, old_style,
for_msgpack)
for key, value in obj.items()
}
def encode_sub(self, validator, value):
# type: (bv.Validator, typing.Any) -> typing.Any
"""
Callback intended to be called by other ``encode`` methods to
delegate encoding of sub-values. Arguments have the same semantics
as with the ``encode`` method.
"""
if isinstance(validator, bv.List):
# Because Lists are mutable, we always validate them during
# serialization
validate_f = validator.validate
encode_f = self.encode_list
elif isinstance(validator, bv.Nullable):
validate_f = validator.validate
encode_f = self.encode_nullable
elif isinstance(validator, bv.Primitive):
validate_f = validator.validate
encode_f = self.encode_primitive
elif isinstance(validator, bv.Struct):
if isinstance(validator, bv.StructTree):
validate_f = validator.validate
encode_f = self.encode_struct_tree
else:
# Fields are already validated on assignment
validate_f = validator.validate_type_only
encode_f = self.encode_struct
elif isinstance(validator, bv.Union):
# Fields are already validated on assignment
validate_f = validator.validate_type_only
encode_f = self.encode_union
else:
raise bv.ValidationError('Unsupported data type {}'.format(
type(validator).__name__))
validate_f(value)
return encode_f(validator, value)
def decode_union_dict(self, data_type, obj):
if '.tag' not in obj:
raise bv.ValidationError("missing '.tag' key")
tag = obj['.tag']
if not isinstance(tag, six.string_types):
raise bv.ValidationError('tag must be string, got %s' %
bv.generic_type_name(tag))
if not data_type.definition._is_tag_present(tag,
self.caller_permissions):
if not self.strict and data_type.definition._catch_all:
return data_type.definition._catch_all, None
else:
raise bv.ValidationError("unknown tag '%s'" % tag)
if tag == data_type.definition._catch_all:
raise bv.ValidationError(
"unexpected use of the catch-all tag '%s'" % tag)
val_data_type = data_type.definition._get_val_data_type(
tag, self.caller_permissions)
if isinstance(val_data_type, bv.Nullable):
val_data_type = val_data_type.validator
nullable = True
else:
nullable = False
if isinstance(val_data_type, bv.Void):
if self.strict:
# In strict mode, ensure there are no extraneous keys set. In
# non-strict mode, we accept that other keys may be set due to a
# change of the void type to another.
if tag in obj:
if obj[tag] is not None:
raise bv.ValidationError(
'expected null, got %s' %
bv.generic_type_name(obj[tag]))
for key in obj:
if key != tag and key != '.tag':
raise bv.ValidationError("unexpected key '%s'" % key)
val = None
elif isinstance(
val_data_type,
(bv.Primitive, bv.List, bv.StructTree, bv.Union, bv.Map)):
if tag in obj:
raw_val = obj[tag]
try:
val = self.json_compat_obj_decode_helper(
val_data_type, raw_val)
except bv.ValidationError as e:
e.add_parent(tag)
raise
else:
# Check no other keys
if nullable:
val = None
else:
raise bv.ValidationError("missing '%s' key" % tag)
for key in obj:
if key != tag and key != '.tag':
raise bv.ValidationError("unexpected key '%s'" % key)
elif isinstance(val_data_type, bv.Struct):
if nullable and len(obj) == 1: # only has a .tag key
val = None
else:
# assume it's not null
raw_val = obj
try:
val = self.json_compat_obj_decode_helper(
val_data_type, raw_val)
except bv.ValidationError as e:
e.add_parent(tag)
raise
else:
assert False, type(val_data_type)
return tag, val
def _decode_union_dict(data_type, obj, alias_validators, strict, for_msgpack):
if '.tag' not in obj:
raise bv.ValidationError("missing '.tag' key")
tag = obj['.tag']
if not isinstance(tag, six.string_types):
raise bv.ValidationError(
'tag must be string, got %s' % bv.generic_type_name(tag))
if tag not in data_type.definition._tagmap:
if not strict and data_type.definition._catch_all:
return data_type.definition._catch_all, None
else:
raise bv.ValidationError("unknown tag '%s'" % tag)
if tag == data_type.definition._catch_all:
raise bv.ValidationError(
"unexpected use of the catch-all tag '%s'" % tag)
val_data_type = data_type.definition._tagmap[tag]
if isinstance(val_data_type, bv.Nullable):
val_data_type = val_data_type.validator
nullable = True
else:
nullable = False
if isinstance(val_data_type, bv.Void):
if tag in obj:
if obj[tag] is not None:
raise bv.ValidationError('expected null, got %s' %
bv.generic_type_name(obj[tag]))
for key in obj:
if key != tag and key != '.tag':
raise bv.ValidationError("unexpected key '%s'" % key)
val = None
elif isinstance(val_data_type,
(bv.Primitive, bv.List, bv.StructTree, bv.Union)):
if tag in obj:
raw_val = obj[tag]
try:
val = _json_compat_obj_decode_helper(
val_data_type, raw_val, alias_validators, strict, False, for_msgpack)
except bv.ValidationError as e:
e.add_parent(tag)
raise
else:
# Check no other keys
if nullable:
val = None
else:
raise bv.ValidationError("missing '%s' key" % tag)
for key in obj:
if key != tag and key != '.tag':
raise bv.ValidationError("unexpected key '%s'" % key)
elif isinstance(val_data_type, bv.Struct):
if nullable and len(obj) == 1: # only has a .tag key
val = None
else:
# assume it's not null
raw_val = obj
try:
val = _json_compat_obj_decode_helper(
val_data_type, raw_val, alias_validators, strict, False,
for_msgpack)
except bv.ValidationError as e:
e.add_parent(tag)
raise
else:
assert False, type(val_data_type)
return tag, val
