def encode_union(self, validator, value): if value._tag is None: raise bv.ValidationError('no tag set') if not validator.definition._is_tag_present(value._tag, self.caller_permissions): raise bv.ValidationError( "caller does not have access to '{}' tag".format(value._tag)) field_validator = validator.definition._get_val_data_type( value._tag, self.caller_permissions) is_none = isinstance(field_validator, bv.Void) \ or (isinstance(field_validator, bv.Nullable) and value._value is None) def encode_sub(sub_validator, sub_value, parent_tag): try: encoded_val = self.encode_sub(sub_validator, sub_value) except bv.ValidationError as exc: exc.add_parent(parent_tag) raise else: return encoded_val if self.old_style: if field_validator is None: return value._tag elif is_none: return value._tag else: encoded_val = encode_sub(field_validator, value._value, value._tag) return {value._tag: encoded_val} elif is_none: return {'.tag': value._tag} else: encoded_val = encode_sub(field_validator, value._value, value._tag) if isinstance(field_validator, bv.Nullable): # We've already checked for the null case above, # so now we're only interested in what the # wrapped validator is field_validator = field_validator.validator if isinstance(field_validator, bv.Struct) \ and not isinstance(field_validator, bv.StructTree): d = collections.OrderedDict( ) # type: typing.Dict[str, typing.Any] d['.tag'] = value._tag d.update(encoded_val) return d else: return collections.OrderedDict(( ('.tag', value._tag), (value._tag, encoded_val), ))
def decode_struct(self, data_type, obj): """ The data_type argument must be a Struct. See json_compat_obj_decode() for argument descriptions. """ if obj is None and data_type.has_default(): return data_type.get_default() elif not isinstance(obj, dict): raise bv.ValidationError('expected object, got %s' % bv.generic_type_name(obj)) all_fields = data_type.definition._all_fields_ for extra_permission in self.caller_permissions.permissions: all_extra_fields = '_all_{}_fields_'.format(extra_permission) all_fields = all_fields + getattr(data_type.definition, all_extra_fields, []) if self.strict: all_field_names = data_type.definition._all_field_names_ for extra_permission in self.caller_permissions.permissions: all_extra_field_names = '_all_{}_field_names_'.format( extra_permission) all_field_names = all_field_names.union( getattr(data_type.definition, all_extra_field_names, {})) for key in obj: if (key not in all_field_names and not key.startswith('.tag')): raise bv.ValidationError("unknown field '%s'" % key) ins = data_type.definition() self.decode_struct_fields(ins, all_fields, obj) # Check that all required fields have been set. data_type.validate_fields_only_with_permissions( ins, self.caller_permissions) return ins
def _encode_union_old(data_type, obj, alias_validators, for_msgpack): """ The data_type argument must be a Union. See json_encode() for argument descriptions. """ if obj._tag is None: raise bv.ValidationError('no tag set') field_data_type = data_type.definition._tagmap[obj._tag] if field_data_type is None: return obj._tag else: if (isinstance(field_data_type, bv.Void) or (isinstance(field_data_type, bv.Nullable) and obj._value is None)): return obj._tag else: try: encoded_val = _json_compat_obj_encode_helper( field_data_type, obj._value, alias_validators, True, for_msgpack) except bv.ValidationError as e: e.add_parent(obj._tag) raise else: return {obj._tag: encoded_val}
def _encode_union(data_type, obj, alias_validators, for_msgpack): """ The data_type argument must be a Union. See json_encode() for argument descriptions. """ if obj._tag is None: raise bv.ValidationError('no tag set') field_data_type = data_type.definition._tagmap[obj._tag] if (isinstance(field_data_type, bv.Void) or (isinstance(field_data_type, bv.Nullable) and obj._value is None)): return {'.tag': obj._tag} else: try: encoded_val = _json_compat_obj_encode_helper( field_data_type, obj._value, alias_validators, False, for_msgpack) except bv.ValidationError as e: e.add_parent(obj._tag) raise else: if isinstance(field_data_type, bv.Nullable): # We've already checked for the null case above, so now we're # only interested in what the wrapped validator is. field_data_type = field_data_type.validator if (isinstance(field_data_type, bv.Struct) and not isinstance(field_data_type, bv.StructTree)): d = collections.OrderedDict() d['.tag'] = obj._tag d.update(encoded_val) return d else: return collections.OrderedDict([ ('.tag', obj._tag), (obj._tag, encoded_val)])
def _encode_struct(data_type, obj, alias_validators, old_style, for_msgpack): """ The data_type argument must be a Struct or StructTree. See json_encode() for argument descriptions. """ # We skip validation of fields with primitive data types in structs and # unions because they've already been validated on assignment. d = collections.OrderedDict() for field_name, field_data_type in data_type.definition._all_fields_: try: val = getattr(obj, field_name) except AttributeError as e: raise bv.ValidationError(e.args[0]) presence_key = '_%s_present' % field_name if val is not None and getattr(obj, presence_key): # This check makes sure that we don't serialize absent struct # fields as null, even if there is a default. try: d[field_name] = _json_compat_obj_encode_helper( field_data_type, val, alias_validators, old_style, for_msgpack) except bv.ValidationError as e: e.add_parent(field_name) raise return d
def encode_struct(self, validator, value): # Skip validation of fields with primitive data types because # they've already been validated on assignment d = collections.OrderedDict() # type: typing.Dict[str, typing.Any] all_fields = validator.definition._all_fields_ for extra_permission in self.caller_permissions.permissions: all_fields_name = '_all_{}_fields_'.format(extra_permission) all_fields = all_fields + getattr(validator.definition, all_fields_name, []) for field_name, field_validator in all_fields: try: field_value = getattr(value, field_name) except AttributeError as exc: raise bv.ValidationError(exc.args[0]) presence_key = '_%s_present' % field_name if field_value is not None \ and getattr(value, presence_key): # Only serialize struct fields that have been explicitly # set, even if there is a default try: d[field_name] = self.encode_sub(field_validator, field_value) except bv.ValidationError as exc: exc.add_parent(field_name) raise return d
def encode_sub(self, validator, value): # type: (bv.Validator, typing.Any) -> typing.Any """ Callback intended to be called by other ``encode`` methods to delegate encoding of sub-values. Arguments have the same semantics as with the ``encode`` method. """ if isinstance(validator, bv.List): # Because Lists are mutable, we always validate them during # serialization validate_f = validator.validate # type: typing.Callable[[typing.Any], None] encode_f = self.encode_list # type: typing.Callable[[typing.Any, typing.Any], typing.Any] # noqa: E501 elif isinstance(validator, bv.Map): # Also validate maps during serialization because they are also mutable validate_f = validator.validate encode_f = self.encode_map elif isinstance(validator, bv.Nullable): validate_f = validator.validate encode_f = self.encode_nullable elif isinstance(validator, bv.Primitive): validate_f = validator.validate encode_f = self.encode_primitive elif isinstance(validator, bv.Struct): if isinstance(validator, bv.StructTree): if self.caller_permissions.permissions: def validate_with_permissions(val): validator.validate_with_permissions( val, self.caller_permissions) validate_f = validate_with_permissions else: validate_f = validator.validate encode_f = self.encode_struct_tree else: # Fields are already validated on assignment if self.caller_permissions.permissions: def validate_with_permissions(val): validator.validate_with_permissions( val, self.caller_permissions) validate_f = validate_with_permissions else: validate_f = validator.validate_type_only encode_f = self.encode_struct elif isinstance(validator, bv.Union): # Fields are already validated on assignment validate_f = validator.validate_type_only encode_f = self.encode_union else: raise bv.ValidationError('Unsupported data type {}'.format( type(validator).__name__)) validate_f(value) return encode_f(validator, value)
def make_stone_friendly(self, data_type, val, validate): """ Convert a Python object to a type that will pass validation by its validator. Validation by ``alias_validators`` is performed even if ``validate`` is false. """ if isinstance(data_type, bv.Timestamp): try: ret = datetime.datetime.strptime(val, data_type.format) except: # datetime.datetime.strptime(val, data_type.format) returned NoneType. Trying alterntive pass try: ret = datetime.datetime( *(time.strptime(val, data_type.format)[0:6])) except (TypeError, ValueError) as e: raise bv.ValidationError(e.args[0]) elif isinstance(data_type, bv.Bytes): if self.for_msgpack: if isinstance(val, six.text_type): ret = val.encode('utf-8') else: ret = val else: try: ret = base64.b64decode(val) except TypeError: raise bv.ValidationError('invalid base64-encoded bytes') elif isinstance(data_type, bv.Void): if self.strict and val is not None: raise bv.ValidationError("expected null, got value") return None else: if validate: if self.caller_permissions.permissions: data_type.validate_with_permissions( val, self.caller_permissions) else: data_type.validate(val) ret = val if self.alias_validators is not None and data_type in self.alias_validators: self.alias_validators[data_type](ret) return ret
def _make_stone_friendly(data_type, val, alias_validators, strict, validate, for_msgpack): """ Convert a Python object to a type that will pass validation by its validator. Validation by ``alias_validators`` is performed even if ``validate`` is false. fix found at: https://www.dropboxforum.com/t5/API-support/Upload-Error-with-v2-migration-from-v1/td-p/244561 """ if isinstance(data_type, bv.Timestamp): try: ret = datetime.datetime.strptime(val, data_type.format) except: #print("datetime.datetime.strptime(val, data_type.format) returned NoneType. Trying alterntive") pass try: ret = datetime.datetime( *(time.strptime(val, data_type.format)[0:6])) except (TypeError, ValueError) as e: raise bv.ValidationError(e.args[0]) elif isinstance(data_type, bv.Bytes): if for_msgpack: if isinstance(val, six.text_type): ret = val.encode('utf-8') else: ret = val else: try: ret = base64.b64decode(val) except TypeError: raise bv.ValidationError('invalid base64-encoded bytes') elif isinstance(data_type, bv.Void): if strict and val is not None: raise bv.ValidationError("expected null, got value") return None else: if validate: data_type.validate(val) ret = val if alias_validators is not None and data_type in alias_validators: alias_validators[data_type](ret) return ret
def decode_list(self, data_type, obj): """ The data_type argument must be a List. See json_compat_obj_decode() for argument descriptions. """ if not isinstance(obj, list): raise bv.ValidationError( 'expected list, got %s' % bv.generic_type_name(obj)) return [ self.json_compat_obj_decode_helper(data_type.item_validator, item) for item in obj]
def decode_map(self, data_type, obj): """ The data_type argument must be a Map. See json_compat_obj_decode() for argument descriptions. """ if not isinstance(obj, dict): raise bv.ValidationError( 'expected dict, got %s' % bv.generic_type_name(obj)) return { self.json_compat_obj_decode_helper(data_type.key_validator, key): self.json_compat_obj_decode_helper(data_type.value_validator, value) for key, value in obj.items() }
def _decode_struct(data_type, obj, alias_validators, strict, old_style, for_msgpack): """ The data_type argument must be a Struct. See json_compat_obj_decode() for argument descriptions. """ if obj is None and data_type.has_default(): return data_type.get_default() elif not isinstance(obj, dict): raise bv.ValidationError('expected object, got %s' % bv.generic_type_name(obj)) if strict: for key in obj: if (key not in data_type.definition._all_field_names_ and not key.startswith('.tag')): raise bv.ValidationError("unknown field '%s'" % key) ins = data_type.definition() _decode_struct_fields(ins, data_type.definition._all_fields_, obj, alias_validators, strict, old_style, for_msgpack) # Check that all required fields have been set. data_type.validate_fields_only(ins) return ins
def decode_union_old(self, data_type, obj): """ The data_type argument must be a Union. See json_compat_obj_decode() for argument descriptions. """ val = None if isinstance(obj, six.string_types): # Union member has no associated value tag = obj if data_type.definition._is_tag_present(tag, self.caller_permissions): val_data_type = data_type.definition._get_val_data_type( tag, self.caller_permissions) if not isinstance(val_data_type, (bv.Void, bv.Nullable)): raise bv.ValidationError( "expected object for '%s', got symbol" % tag) else: if not self.strict and data_type.definition._catch_all: tag = data_type.definition._catch_all else: raise bv.ValidationError("unknown tag '%s'" % tag) elif isinstance(obj, dict): # Union member has value if len(obj) != 1: raise bv.ValidationError('expected 1 key, got %s' % len(obj)) tag = list(obj)[0] raw_val = obj[tag] if data_type.definition._is_tag_present(tag, self.caller_permissions): val_data_type = data_type.definition._get_val_data_type( tag, self.caller_permissions) if isinstance(val_data_type, bv.Nullable) and raw_val is None: val = None elif isinstance(val_data_type, bv.Void): if raw_val is None or not self.strict: # If raw_val is None, then this is the more verbose # representation of a void union member. If raw_val isn't # None, then maybe the spec has changed, so check if we're # in strict mode. val = None else: raise bv.ValidationError('expected null, got %s' % bv.generic_type_name(raw_val)) else: try: val = self.json_compat_obj_decode_helper( val_data_type, raw_val) except bv.ValidationError as e: e.add_parent(tag) raise else: if not self.strict and data_type.definition._catch_all: tag = data_type.definition._catch_all else: raise bv.ValidationError("unknown tag '%s'" % tag) else: raise bv.ValidationError("expected string or object, got %s" % bv.generic_type_name(obj)) return data_type.definition(tag, val)
def _make_stone_friendly(data_type, val, alias_validators, strict, validate, for_msgpack): """ Convert a Python object to a type that will pass validation by its validator. Validation by ``alias_validators`` is performed even if ``validate`` is false. """ if isinstance(data_type, bv.Timestamp): try: # modified for kodi's issues with datetime ret = datetime.datetime( *(time.strptime(val, data_type.format)[0:6])) except (TypeError, ValueError) as e: raise bv.ValidationError(e.args[0]) elif isinstance(data_type, bv.Bytes): if for_msgpack: if isinstance(val, six.text_type): ret = val.encode('utf-8') else: ret = val else: try: ret = base64.b64decode(val) except TypeError: raise bv.ValidationError('invalid base64-encoded bytes') elif isinstance(data_type, bv.Void): if strict and val is not None: raise bv.ValidationError("expected null, got value") return None else: if validate: data_type.validate(val) ret = val if alias_validators is not None and data_type in alias_validators: alias_validators[data_type](ret) return ret
def _decode_list( data_type, obj, alias_validators, strict, old_style, for_msgpack): """ The data_type argument must be a List. See json_compat_obj_decode() for argument descriptions. """ if not isinstance(obj, list): raise bv.ValidationError( 'expected list, got %s' % bv.generic_type_name(obj)) return [ _json_compat_obj_decode_helper( data_type.item_validator, item, alias_validators, strict, old_style, for_msgpack) for item in obj]
def determine_struct_tree_subtype(self, data_type, obj): """ Searches through the JSON-object-compatible dict using the data type definition to determine which of the enumerated subtypes `obj` is. """ if '.tag' not in obj: raise bv.ValidationError("missing '.tag' key") if not isinstance(obj['.tag'], six.string_types): raise bv.ValidationError('expected string, got %s' % bv.generic_type_name(obj['.tag']), parent='.tag') # Find the subtype the tags refer to full_tags_tuple = (obj['.tag'], ) if full_tags_tuple in data_type.definition._tag_to_subtype_: subtype = data_type.definition._tag_to_subtype_[full_tags_tuple] if isinstance(subtype, bv.StructTree): raise bv.ValidationError( "tag '%s' refers to non-leaf subtype" % ('.'.join(full_tags_tuple))) return subtype else: if self.strict: # In strict mode, the entirety of the tag hierarchy should # point to a known subtype. raise bv.ValidationError("unknown subtype '%s'" % '.'.join(full_tags_tuple)) else: # If subtype was not found, use the base. if data_type.definition._is_catch_all_: return data_type else: raise bv.ValidationError( "unknown subtype '%s' and '%s' is not a catch-all" % ('.'.join(full_tags_tuple), data_type.definition.__name__))
def json_decode(data_type, serialized_obj, caller_permissions=None, alias_validators=None, strict=True, old_style=False): """Performs the reverse operation of json_encode. Args: data_type (Validator): Validator for serialized_obj. serialized_obj (str): The JSON string to deserialize. caller_permissions (list): The list of raw-string caller permissions with which to serialize. alias_validators (Optional[Mapping[bv.Validator, Callable[[], None]]]): Custom validation functions. These must raise bv.ValidationError on failure. strict (bool): If strict, then unknown struct fields will raise an error, and unknown union variants will raise an error even if a catch all field is specified. strict should only be used by a recipient of serialized JSON if it's guaranteed that its Stone specs are at least as recent as the senders it receives messages from. Returns: The returned object depends on the input data_type. - Boolean -> bool - Bytes -> bytes - Float -> float - Integer -> long - List -> list - Map -> dict - Nullable -> None or its wrapped type. - String -> unicode (PY2) or str (PY3) - Struct -> An instance of its definition attribute. - Timestamp -> datetime.datetime - Union -> An instance of its definition attribute. """ try: deserialized_obj = json.loads(serialized_obj) except ValueError: raise bv.ValidationError('could not decode input as JSON') else: return json_compat_obj_decode(data_type, deserialized_obj, caller_permissions=caller_permissions, alias_validators=alias_validators, strict=strict, old_style=old_style)
def _decode_map(data_type, obj, alias_validators, strict, old_style, for_msgpack): """ The data_type argument must be a Map. See json_compat_obj_decode() for argument descriptions. """ if not isinstance(obj, dict): raise bv.ValidationError('expected dict, got %s' % bv.generic_type_name(obj)) return { _json_compat_obj_decode_helper(data_type.key_validator, key, alias_validators, strict, old_style, for_msgpack): _json_compat_obj_decode_helper(data_type.value_validator, value, alias_validators, strict, old_style, for_msgpack) for key, value in obj.items() }
def encode_sub(self, validator, value): # type: (bv.Validator, typing.Any) -> typing.Any """ Callback intended to be called by other ``encode`` methods to delegate encoding of sub-values. Arguments have the same semantics as with the ``encode`` method. """ if isinstance(validator, bv.List): # Because Lists are mutable, we always validate them during # serialization validate_f = validator.validate encode_f = self.encode_list elif isinstance(validator, bv.Nullable): validate_f = validator.validate encode_f = self.encode_nullable elif isinstance(validator, bv.Primitive): validate_f = validator.validate encode_f = self.encode_primitive elif isinstance(validator, bv.Struct): if isinstance(validator, bv.StructTree): validate_f = validator.validate encode_f = self.encode_struct_tree else: # Fields are already validated on assignment validate_f = validator.validate_type_only encode_f = self.encode_struct elif isinstance(validator, bv.Union): # Fields are already validated on assignment validate_f = validator.validate_type_only encode_f = self.encode_union else: raise bv.ValidationError('Unsupported data type {}'.format( type(validator).__name__)) validate_f(value) return encode_f(validator, value)
def decode_union_dict(self, data_type, obj): if '.tag' not in obj: raise bv.ValidationError("missing '.tag' key") tag = obj['.tag'] if not isinstance(tag, six.string_types): raise bv.ValidationError('tag must be string, got %s' % bv.generic_type_name(tag)) if not data_type.definition._is_tag_present(tag, self.caller_permissions): if not self.strict and data_type.definition._catch_all: return data_type.definition._catch_all, None else: raise bv.ValidationError("unknown tag '%s'" % tag) if tag == data_type.definition._catch_all: raise bv.ValidationError( "unexpected use of the catch-all tag '%s'" % tag) val_data_type = data_type.definition._get_val_data_type( tag, self.caller_permissions) if isinstance(val_data_type, bv.Nullable): val_data_type = val_data_type.validator nullable = True else: nullable = False if isinstance(val_data_type, bv.Void): if self.strict: # In strict mode, ensure there are no extraneous keys set. In # non-strict mode, we accept that other keys may be set due to a # change of the void type to another. if tag in obj: if obj[tag] is not None: raise bv.ValidationError( 'expected null, got %s' % bv.generic_type_name(obj[tag])) for key in obj: if key != tag and key != '.tag': raise bv.ValidationError("unexpected key '%s'" % key) val = None elif isinstance( val_data_type, (bv.Primitive, bv.List, bv.StructTree, bv.Union, bv.Map)): if tag in obj: raw_val = obj[tag] try: val = self.json_compat_obj_decode_helper( val_data_type, raw_val) except bv.ValidationError as e: e.add_parent(tag) raise else: # Check no other keys if nullable: val = None else: raise bv.ValidationError("missing '%s' key" % tag) for key in obj: if key != tag and key != '.tag': raise bv.ValidationError("unexpected key '%s'" % key) elif isinstance(val_data_type, bv.Struct): if nullable and len(obj) == 1: # only has a .tag key val = None else: # assume it's not null raw_val = obj try: val = self.json_compat_obj_decode_helper( val_data_type, raw_val) except bv.ValidationError as e: e.add_parent(tag) raise else: assert False, type(val_data_type) return tag, val
def _decode_union_dict(data_type, obj, alias_validators, strict, for_msgpack): if '.tag' not in obj: raise bv.ValidationError("missing '.tag' key") tag = obj['.tag'] if not isinstance(tag, six.string_types): raise bv.ValidationError( 'tag must be string, got %s' % bv.generic_type_name(tag)) if tag not in data_type.definition._tagmap: if not strict and data_type.definition._catch_all: return data_type.definition._catch_all, None else: raise bv.ValidationError("unknown tag '%s'" % tag) if tag == data_type.definition._catch_all: raise bv.ValidationError( "unexpected use of the catch-all tag '%s'" % tag) val_data_type = data_type.definition._tagmap[tag] if isinstance(val_data_type, bv.Nullable): val_data_type = val_data_type.validator nullable = True else: nullable = False if isinstance(val_data_type, bv.Void): if tag in obj: if obj[tag] is not None: raise bv.ValidationError('expected null, got %s' % bv.generic_type_name(obj[tag])) for key in obj: if key != tag and key != '.tag': raise bv.ValidationError("unexpected key '%s'" % key) val = None elif isinstance(val_data_type, (bv.Primitive, bv.List, bv.StructTree, bv.Union)): if tag in obj: raw_val = obj[tag] try: val = _json_compat_obj_decode_helper( val_data_type, raw_val, alias_validators, strict, False, for_msgpack) except bv.ValidationError as e: e.add_parent(tag) raise else: # Check no other keys if nullable: val = None else: raise bv.ValidationError("missing '%s' key" % tag) for key in obj: if key != tag and key != '.tag': raise bv.ValidationError("unexpected key '%s'" % key) elif isinstance(val_data_type, bv.Struct): if nullable and len(obj) == 1: # only has a .tag key val = None else: # assume it's not null raw_val = obj try: val = _json_compat_obj_decode_helper( val_data_type, raw_val, alias_validators, strict, False, for_msgpack) except bv.ValidationError as e: e.add_parent(tag) raise else: assert False, type(val_data_type) return tag, val