def _deserialize(
self,
value: typing.Any,
attr: typing.Optional[str],
data: typing.Optional[typing.Mapping[str, typing.Any]],
**kwargs,
):
error_store = ErrorStore()
if self.many:
result = []
if utils.is_collection(value):
for collection_entry in value:
result.append(self._check_schemas(collection_entry))
return result
raise self.make_error("type",
input=value,
type=value.__class__.__name__)
for schema in self.nested:
try:
return common.resolve_schema_instance(schema).load(
value, unknown=self.unknown)
except ValidationError as exc:
error_store.store_error(exc.messages,
field_name=exc.field_name)
raise ValidationError(error_store.errors, data=value)
def _load_schemas(self, scalar: Result, partial=None) -> Result:
rv = {}
error_store = ErrorStore()
value = dict(scalar)
for schema in self._nested_schemas():
schema_inst = common.resolve_schema_instance(schema)
try:
loaded = schema_inst.load(
value,
unknown=self.unknown,
partial=partial,
)
if not self.merged:
return loaded
for key in schema_inst.declared_fields:
if key in value:
del value[key]
except ValidationError as exc:
for key in schema_inst.declared_fields:
if key in value:
del value[key]
error_store.store_error({exc.field_name: exc.messages})
continue
if self.merged:
rv.update(loaded)
if error_store.errors:
raise ValidationError(error_store.errors)
return rv
def _serialize(
self,
value: typing.Any,
attr: str,
obj: typing.Any,
**kwargs,
) -> typing.Union[Result, list[Result]]:
result: typing.Any
error_store = ErrorStore()
result = []
if self.many:
if utils.is_collection(value):
for entry in value:
result.extend(self._dump_schemas(entry))
else:
error_store.store_error(self._make_type_error(value))
else:
result.extend(self._dump_schemas(value))
if error_store.errors:
raise ValidationError(error_store.errors, data=value)
if self.merged and not self.many:
rv = {}
for entry in result:
for key, _value in entry.items():
if key in rv:
raise ValidationError({key: "Can't collate result. Key occurs twice."})
rv[key] = _value
return rv
return result
def _dump_schemas(self, scalar: Result) -> list[Result]:
rv = []
error_store = ErrorStore()
value = dict(scalar)
for schema in self._nested_schemas():
schema_inst = common.resolve_schema_instance(schema)
try:
dumped = schema_inst.dump(value, many=False)
if not self.merged:
return dumped
loaded = schema_inst.load(dumped)
# We check what could actually pass through the load() call, because some
# schemas validate keys without having them defined in their _declared_fields.
for key in loaded.keys():
if key in value:
del value[key]
except ValidationError as exc:
# When we encounter an error, we can't do anything besides remove the keys, which
# we know about.
for key in schema_inst.declared_fields:
if key in value:
del value[key]
error_store.store_error({exc.field_name: exc.messages})
continue
if not isinstance(schema_inst, MultiNested.ValidateOnDump):
rv.append(dumped)
if error_store.errors:
raise ValidationError(error_store.errors)
return rv
def _check_schemas(self, scalar, partial=None):
error_store = ErrorStore()
for schema in self.nested:
try:
return common.resolve_schema_instance(schema).load(
scalar,
unknown=self.unknown,
partial=partial,
)
except ValidationError as exc:
error_store.store_error(exc.messages,
field_name=exc.field_name)
raise ValidationError(error_store.errors, data=scalar)
def dump(self, obj, *, many=None):
"""Serialize an object to native Python data types according to this
Schema's fields.
:param obj: The object to serialize.
:param bool many: Whether to serialize `obj` as a collection. If `None`, the value
for `self.many` is used.
:return: A dict of serialized data
:rtype: dict
.. versionadded:: 1.0.0
.. versionchanged:: 3.0.0b7
This method returns the serialized data rather than a ``(data, errors)`` duple.
A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised
if ``obj`` is invalid.
"""
error_store = ErrorStore()
errors = {}
many = self.many if many is None else bool(many)
if many and is_iterable_but_not_string(obj):
obj = list(obj)
if self._has_processors(PRE_DUMP):
try:
processed_obj = self._invoke_dump_processors(
PRE_DUMP, obj, many=many, original_data=obj
)
except ValidationError as error:
errors = error.normalized_messages()
result = None
else:
processed_obj = obj
if not errors:
result = self._serialize(
processed_obj,
fields_dict=self.fields,
error_store=error_store,
many=many,
accessor=self.get_attribute,
dict_class=self.dict_class,
index_errors=self.opts.index_errors,
)
errors = error_store.errors
if not errors and self._has_processors(POST_DUMP):
try:
result = self._invoke_dump_processors(
POST_DUMP, result, many=many, original_data=obj
)
except ValidationError as error:
errors = error.normalized_messages()
if errors:
exc = ValidationError(errors, data=obj, valid_data=result)
# User-defined error handler
self.handle_error(exc, obj, many=many)
raise exc
return result
def _serialize(
self,
value: typing.Any,
attr: str,
obj: typing.Any,
**kwargs,
):
if value is None:
return None
error_store = ErrorStore()
for schema in self.nested:
try:
return common.resolve_schema_instance(schema).dump(
value, many=self.many)
except ValidationError as exc:
error_store.store_error(exc.messages,
field_name=exc.field_name)
raise ValidationError(error_store.errors, data=value)
def validate_only(self, data):
"""
Bypass deserialization and just run field validators. This is taken
from the marshmallow _do_load function:
https://github.com/marshmallow-code/marshmallow/blob/3.5.2/src/marshmallow/schema.py#L807
"""
error_store = ErrorStore()
# Run field-level validation
self._invoke_field_validators(error_store=error_store,
data=data,
many=None)
# Run schema-level validation
if self._has_processors(decorators.VALIDATES_SCHEMA):
field_errors = bool(error_store.errors)
self._invoke_schema_validators(
error_store=error_store,
pass_many=True,
data=data,
original_data=data,
many=None,
partial=None,
field_errors=field_errors,
)
self._invoke_schema_validators(
error_store=error_store,
pass_many=False,
data=data,
original_data=data,
many=None,
partial=None,
field_errors=field_errors,
)
errors = error_store.errors
if errors:
exc = MarshmallowValidationError(errors,
data=data,
valid_data=data)
self.handle_error(exc, data, many=None, partial=None)
raise exc
return data
def _do_load(
self,
data: typing.Union[
typing.Mapping[str, typing.Any],
typing.Iterable[typing.Mapping[str, typing.Any]],
],
*,
many: bool = None,
partial: typing.Union[bool, types.StrSequenceOrSet] = None,
unknown: str = None,
postprocess: bool = True
):
"""Deserialize `data`, returning the deserialized result.
This method is private API.
:param data: The data to deserialize.
:param many: Whether to deserialize `data` as a collection. If `None`, the
value for `self.many` is used.
:param partial: Whether to validate required fields. If its
value is an iterable, only fields listed in that iterable will be
ignored will be allowed missing. If `True`, all fields will be allowed missing.
If `None`, the value for `self.partial` is used.
:param unknown: Whether to exclude, include, or raise an error for unknown
fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.
If `None`, the value for `self.unknown` is used.
:param postprocess: Whether to run post_load methods..
:return: Deserialized data
"""
error_store = ErrorStore()
errors = {} # type: typing.Dict[str, typing.List[str]]
many = self.many if many is None else bool(many)
unknown = unknown or self.unknown
if partial is None:
partial = self.partial
# Run preprocessors
if self._has_processors(PRE_LOAD):
try:
processed_data = self._invoke_load_processors(
PRE_LOAD, data, many=many, original_data=data, partial=partial
)
except ValidationError as err:
errors = err.normalized_messages()
result = (
None
) # type: typing.Optional[typing.Union[typing.List, typing.Dict]]
else:
processed_data = data
if not errors:
# Deserialize data
result = self._deserialize(
processed_data,
error_store=error_store,
many=many,
partial=partial,
unknown=unknown,
)
# Run field-level validation
self._invoke_field_validators(
error_store=error_store, data=result, many=many
)
# Run schema-level validation
if self._has_processors(VALIDATES_SCHEMA):
field_errors = bool(error_store.errors)
self._invoke_schema_validators(
error_store=error_store,
pass_many=True,
data=result,
original_data=data,
many=many,
partial=partial,
field_errors=field_errors,
)
self._invoke_schema_validators(
error_store=error_store,
pass_many=False,
data=result,
original_data=data,
many=many,
partial=partial,
field_errors=field_errors,
)
errors = error_store.errors
# Run post processors
if not errors and postprocess and self._has_processors(POST_LOAD):
try:
result = self._invoke_load_processors(
POST_LOAD,
result,
many=many,
original_data=data,
partial=partial,
)
except ValidationError as err:
errors = err.normalized_messages()
if errors:
exc = ValidationError(errors, data=data, valid_data=result)
self.handle_error(exc, data, many=many, partial=partial)
raise exc
return result
def _deserialize(
self,
data: typing.Union[
typing.Mapping[str, typing.Any],
typing.Iterable[typing.Mapping[str, typing.Any]],
],
*,
error_store: ErrorStore,
many: bool = False,
partial=False,
unknown=RAISE,
index=None
) -> typing.Union[_T, typing.List[_T]]:
"""Deserialize ``data``.
:param dict data: The data to deserialize.
:param ErrorStore error_store: Structure to store errors.
:param bool many: `True` if ``data`` should be deserialized as a collection.
:param bool|tuple partial: Whether to ignore missing fields and not require
any fields declared. Propagates down to ``Nested`` fields as well. If
its value is an iterable, only missing fields listed in that iterable
will be ignored. Use dot delimiters to specify nested fields.
:param unknown: Whether to exclude, include, or raise an error for unknown
fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.
:param int index: Index of the item being serialized (for storing errors) if
serializing a collection, otherwise `None`.
:return: A dictionary of the deserialized data.
"""
index_errors = self.opts.index_errors
index = index if index_errors else None
if many:
if not is_collection(data):
error_store.store_error([self.error_messages["type"]], index=index)
ret = [] # type: typing.List[_T]
else:
ret = [
typing.cast(
_T,
self._deserialize(
typing.cast(typing.Mapping[str, typing.Any], d),
error_store=error_store,
many=False,
partial=partial,
unknown=unknown,
index=idx,
),
)
for idx, d in enumerate(data)
]
return ret
ret = self.dict_class()
# Check data is a dict
if not isinstance(data, Mapping):
error_store.store_error([self.error_messages["type"]], index=index)
else:
partial_is_collection = is_collection(partial)
for attr_name, field_obj in self.load_fields.items():
field_name = (
field_obj.data_key if field_obj.data_key is not None else attr_name
)
raw_value = data.get(field_name, missing)
if raw_value is missing:
# Ignore missing field if we're allowed to.
if partial is True or (
partial_is_collection and attr_name in partial
):
continue
d_kwargs = {}
# Allow partial loading of nested schemas.
if partial_is_collection:
prefix = field_name + "."
len_prefix = len(prefix)
sub_partial = [
f[len_prefix:] for f in partial if f.startswith(prefix)
]
d_kwargs["partial"] = sub_partial
else:
d_kwargs["partial"] = partial
getter = lambda val: field_obj.deserialize(
val, field_name, data, **d_kwargs
)
value = self._call_and_store(
getter_func=getter,
data=raw_value,
field_name=field_name,
error_store=error_store,
index=index,
)
if value is not missing:
key = field_obj.attribute or attr_name
set_value(typing.cast(typing.Dict, ret), key, value)
if unknown != EXCLUDE:
fields = {
field_obj.data_key if field_obj.data_key is not None else field_name
for field_name, field_obj in self.load_fields.items()
}
for key in set(data) - fields:
value = data[key]
if unknown == INCLUDE:
set_value(typing.cast(typing.Dict, ret), key, value)
elif unknown == RAISE:
error_store.store_error(
[self.error_messages["unknown"]],
key,
(index if index_errors else None),
)
return ret
def _deserialize(
self,
data: typing.Union[typing.Mapping[str, typing.Any],
typing.Iterable[typing.Mapping[str, typing.Any]], ],
*args,
error_store: ErrorStore,
many: bool = False,
partial=False,
unknown=RAISE,
index=None,
) -> typing.Union[_T, typing.List[_T]]:
index_errors = self.opts.index_errors
index = index if index_errors else None
if self.flattened and is_collection(data) and not self._all_objects:
new_data = []
self._all_objects = {}
for d in data:
self._all_objects[d["@id"]] = d
if self._compare_ids(d["@type"], self.opts.rdf_type):
new_data.append(d)
data = new_data
if len(data) == 1:
data = data[0]
if many:
if not is_collection(data):
error_store.store_error([self.error_messages["type"]],
index=index)
ret = [] # type: typing.List[_T]
else:
ret = [
typing.cast(
_T,
self._deserialize(
typing.cast(typing.Mapping[str, typing.Any], d),
error_store=error_store,
many=False,
partial=partial,
unknown=unknown,
index=idx,
),
) for idx, d in enumerate(data)
]
return ret
ret = self.dict_class()
# Check data is a dict
if not isinstance(data, Mapping):
error_store.store_error([self.error_messages["type"]], index=index)
else:
if data.get("@context", None):
# we got compacted jsonld, expand it
data = jsonld.expand(data)
partial_is_collection = is_collection(partial)
for attr_name, field_obj in self.load_fields.items():
field_name = field_obj.data_key if field_obj.data_key is not None else attr_name
if getattr(field_obj, "reverse", False):
raw_value = data.get("@reverse", missing)
if raw_value is not missing:
raw_value = raw_value.get(field_name, missing)
elif self.flattened:
# find an object that refers to this one with the same property
raw_value = self.get_reverse_links(data, field_name)
if not raw_value:
raw_value = missing
else:
raw_value = data.get(field_name, missing)
if raw_value is missing:
# Ignore missing field if we're allowed to.
if partial is True or (partial_is_collection
and attr_name in partial):
continue
d_kwargs = {}
# Allow partial loading of nested schemes.
if partial_is_collection:
prefix = field_name + "."
len_prefix = len(prefix)
sub_partial = [
f[len_prefix:] for f in partial if f.startswith(prefix)
]
d_kwargs["partial"] = sub_partial
else:
d_kwargs["partial"] = partial
d_kwargs["_all_objects"] = self._all_objects
d_kwargs["flattened"] = self.flattened
getter = lambda val: field_obj.deserialize(
val, field_name, data, **d_kwargs)
value = self._call_and_store(
getter_func=getter,
data=raw_value,
field_name=field_name,
error_store=error_store,
index=index,
)
if value is not missing:
key = field_obj.attribute or attr_name
set_value(typing.cast(typing.Dict, ret), key, value)
if unknown != EXCLUDE:
fields = {
field_obj.data_key
if field_obj.data_key is not None else field_name
for field_name, field_obj in self.load_fields.items()
}
for key in set(data) - fields:
if key in ["@type", "@reverse"]:
# ignore JsonLD meta fields
continue
value = data[key]
if unknown == INCLUDE:
set_value(typing.cast(typing.Dict, ret), key, value)
elif unknown == RAISE:
error_store.store_error(
[self.error_messages["unknown"]],
key,
(index if index_errors else None),
)
return ret
def _do_load(self,
data,
*,
many=None,
partial=None,
unknown=None,
postprocess=True):
"""Deserialize `data`, returning the deserialized result.
:param data: The data to deserialize.
:param bool many: Whether to deserialize `data` as a collection. If `None`, the
value for `self.many` is used.
:param bool|tuple partial: Whether to validate required fields. If its value is an iterable,
only fields listed in that iterable will be ignored will be allowed missing.
If `True`, all fields will be allowed missing.
If `None`, the value for `self.partial` is used.
:param unknown: Whether to exclude, include, or raise an error for unknown
fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.
If `None`, the value for `self.unknown` is used.
:param bool postprocess: Whether to run post_load methods..
:return: A dict of deserialized data
:rtype: dict
"""
error_store = ErrorStore()
errors = {}
many = self.many if many is None else bool(many)
unknown = unknown or self.unknown
if partial is None:
partial = self.partial
# Run preprocessors
if self._has_processors(PRE_LOAD):
try:
processed_data = self._invoke_load_processors(
PRE_LOAD,
data,
many=many,
original_data=data,
partial=partial,
)
except ValidationError as err:
errors = err.normalized_messages()
result = None
else:
processed_data = data
if not errors:
# Deserialize data
result = self._deserialize(
processed_data,
fields_dict=self.fields,
error_store=error_store,
many=many,
partial=partial,
unknown=unknown,
dict_class=self.dict_class,
index_errors=self.opts.index_errors,
)
# Run field-level validation
self._invoke_field_validators(error_store=error_store,
data=result,
many=many)
# Run schema-level validation
if self._has_processors(VALIDATES_SCHEMA):
field_errors = bool(error_store.errors)
self._invoke_schema_validators(
error_store=error_store,
pass_many=True,
data=result,
original_data=data,
many=many,
partial=partial,
field_errors=field_errors,
)
self._invoke_schema_validators(
error_store=error_store,
pass_many=False,
data=result,
original_data=data,
many=many,
partial=partial,
field_errors=field_errors,
)
errors = error_store.errors
# Run post processors
if not errors and postprocess and self._has_processors(POST_LOAD):
try:
result = self._invoke_load_processors(
POST_LOAD,
result,
many=many,
original_data=data,
partial=partial,
)
except ValidationError as err:
errors = err.normalized_messages()
if errors:
exc = ValidationError(
errors,
data=data,
valid_data=result,
)
self.handle_error(exc, data)
raise exc
return result
