def parse_json(input_data, deserialize_function=None, **kwargs):
"""De-serialize JSON data into a Python :class:`dict <python:dict>` object.
:param input_data: The JSON data to de-serialize.
:type input_data: :class:`str <python:str>`
:param deserialize_function: Optionally override the default JSON deserializer.
Defaults to :obj:`None <python:None>`, which calls the default
:ref:`simplejson.loads() <simplejson:simplejson.loads>`
function from the `simplejson <https://github.com/simplejson/simplejson>`_ library.
.. note::
Use the ``deserialize_function`` parameter to override the default
YAML deserializer. A valid ``deserialize_function`` is expected to
accept a single :class:`str <python:str>` and return a
:class:`dict <python:dict>`, similar to
:ref:`simplejson.loads() <simplejson:simplejson.loads>`
If you wish to pass additional arguments to your ``deserialize_function``
pass them as keyword arguments (in ``kwargs``).
:type deserialize_function: callable / :obj:`None <python:None>`
:param kwargs: Optional keyword parameters that are passed to the
JSON deserializer function. By default, these are options which are passed
to :ref:`simplejson.loads() <simplejson:simplejson.loads>`.
:type kwargs: keyword arguments
:returns: A :class:`dict <python:dict>` representation of ``input_data``.
:rtype: :class:`dict <python:dict>`
"""
is_file = False
if checkers.is_file(input_data):
is_file = True
if deserialize_function is None and not is_file:
deserialize_function = json.loads
elif deserialize_function is None and is_file:
deserialize_function = json.load
else:
if checkers.is_callable(deserialize_function) is False:
raise ValueError('deserialize_function (%s) is not callable' %
deserialize_function)
if not input_data:
raise DeserializationError('input_data is empty')
if not is_file:
try:
input_data = validators.string(input_data, allow_empty=False)
except ValueError:
raise DeserializationError('input_data is not a valid string')
from_json = deserialize_function(input_data, **kwargs)
else:
with open(input_data, 'r') as input_file:
from_json = deserialize_function(input_file, **kwargs)
return from_json
def parse_yaml(input_data, deserialize_function=None, **kwargs):
"""De-serialize YAML data into a Python :class:`dict <python:dict>` object.
:param input_data: The YAML data to de-serialize.
:type input_data: :class:`str <python:str>`
:param deserialize_function: Optionally override the default YAML deserializer.
Defaults to :obj:`None <python:None>`, which calls the default ``yaml.safe_load()``
function from the `PyYAML <https://github.com/yaml/pyyaml>`_ library.
.. note::
Use the ``deserialize_function`` parameter to override the default
YAML deserializer. A valid ``deserialize_function`` is expected to
accept a single :class:`str <python:str>` and return a
:class:`dict <python:dict>`, similar to ``yaml.safe_load()``.
If you wish to pass additional arguments to your ``deserialize_function``
pass them as keyword arguments (in ``kwargs``).
:type deserialize_function: callable / :obj:`None <python:None>`
:param kwargs: Optional keyword parameters that are passed to the
YAML deserializer function. By default, these are options which are passed
to ``yaml.safe_load()``.
:type kwargs: keyword arguments
:returns: A :class:`dict <python:dict>` representation of ``input_data``.
:rtype: :class:`dict <python:dict>`
"""
if deserialize_function is None:
deserialize_function = yaml.safe_load
else:
if checkers.is_callable(deserialize_function) is False:
raise ValueError('deserialize_function (%s) is not callable' %
deserialize_function)
if not input_data:
raise DeserializationError('input_data is empty')
try:
input_data = validators.string(input_data, allow_empty=False)
except ValueError:
raise DeserializationError('input_data is not a valid string')
from_yaml = yaml.safe_load(input_data, **kwargs)
return from_yaml
def parse_csv(input_data,
delimiter='|',
wrap_all_strings=False,
null_text='None',
wrapper_character="'",
double_wrapper_character_when_nested=False,
escape_character="\\",
line_terminator='\r\n'):
"""De-serialize CSV data into a Python :class:`dict <python:dict>` object.
.. versionadded:: 0.3.0
.. tip::
Unwrapped empty column values are automatically interpreted as null
(:obj:`None <python:None>`).
:param input_data: The CSV data to de-serialize. Should include column headers
and at least **one** row of data. Will ignore any rows of data beyond the
first row.
:type input_data: :class:`str <python:str>`
:param delimiter: The delimiter used between columns. Defaults to ``|``.
:type delimiter: :class:`str <python:str>`
:param wrapper_character: The string used to wrap string values when
wrapping is applied. Defaults to ``'``.
:type wrapper_character: :class:`str <python:str>`
:param null_text: The string used to indicate an empty value if empty
values are wrapped. Defaults to `None`.
:type null_text: :class:`str <python:str>`
:returns: A :class:`dict <python:dict>` representation of the CSV record.
:rtype: :class:`dict <python:dict>`
:raises DeserializationError: if ``input_data`` is not a valid
:class:`str <python:str>`
:raises CSVStructureError: if there are less than 2 (two) rows in ``input_data``
or if column headers are not valid Python variable names
"""
use_file = False
if not checkers.is_file(input_data) and not checkers.is_iterable(
input_data):
try:
input_data = validators.string(input_data, allow_empty=False)
except (ValueError, TypeError):
raise DeserializationError("input_data expects a 'str', received '%s'" \
% type(input_data))
input_data = [input_data]
elif checkers.is_file(input_data):
use_file = True
if not wrapper_character:
wrapper_character = '\''
if wrap_all_strings:
quoting = csv.QUOTE_NONNUMERIC
else:
quoting = csv.QUOTE_MINIMAL
if 'sqlathanor' in csv.list_dialects():
csv.unregister_dialect('sqlathanor')
csv.register_dialect('sqlathanor',
delimiter=delimiter,
doublequote=double_wrapper_character_when_nested,
escapechar=escape_character,
quotechar=wrapper_character,
quoting=quoting,
lineterminator=line_terminator)
if not use_file:
csv_reader = csv.DictReader(input_data,
dialect='sqlathanor',
restkey=None,
restval=None)
rows = [x for x in csv_reader]
else:
if not is_py2:
with open(input_data, 'r', newline='') as input_file:
csv_reader = csv.DictReader(input_file,
dialect='sqlathanor',
restkey=None,
restval=None)
rows = [x for x in csv_reader]
else:
with open(input_data, 'r') as input_file:
csv_reader = csv.DictReader(input_file,
dialect='sqlathanor',
restkey=None,
restval=None)
rows = [x for x in csv_reader]
if len(rows) < 1:
raise CSVStructureError(
'expected 1 row of data and 1 header row, missing 1')
else:
data = rows[0]
for key in data:
try:
validators.variable_name(key)
except ValueError:
raise CSVStructureError(
'column (%s) is not a valid Python variable name' % key)
if data[key] == null_text:
data[key] = None
csv.unregister_dialect('sqlathanor')
return data
def _parse_csv(cls,
csv_data,
delimiter='|',
wrap_all_strings=False,
null_text='None',
wrapper_character="'",
double_wrapper_character_when_nested=False,
escape_character="\\",
line_terminator='\r\n',
config_set=None):
"""Generate a :class:`dict <python:dict>` from a CSV record.
.. tip::
Unwrapped empty column values are automatically interpreted as null
(:obj:`None <python:None>`).
:param csv_data: The CSV record. Should be a single row and should **not**
include column headers.
:type csv_data: :class:`str <python:str>`
:param delimiter: The delimiter used between columns. Defaults to ``|``.
:type delimiter: :class:`str <python:str>`
:param wrapper_character: The string used to wrap string values when
wrapping is applied. Defaults to ``'``.
:type wrapper_character: :class:`str <python:str>`
:param null_text: The string used to indicate an empty value if empty
values are wrapped. Defaults to `None`.
:type null_text: :class:`str <python:str>`
:param config_set: If not :obj:`None <python:None>`, the named configuration set
to use. Defaults to :obj:`None <python:None>`.
:type config_set: :class:`str <python:str>` / :obj:`None <python:None>`
:returns: A :class:`dict <python:dict>` representation of the CSV record.
:rtype: :class:`dict <python:dict>`
:raises DeserializationError: if ``csv_data`` is not a valid
:class:`str <python:str>`
:raises CSVStructureError: if the columns in ``csv_data`` do not match
the expected columns returned by
:func:`get_csv_column_names() <BaseModel.get_csv_column_names>`
:raises ValueDeserializationError: if a value extracted from the CSV
failed when executing its :term:`de-serialization function`.
"""
try:
csv_data = validators.string(csv_data, allow_empty=False)
except (ValueError, TypeError):
raise DeserializationError("csv_data expects a 'str', received '%s'" \
% type(csv_data))
if not wrapper_character:
wrapper_character = '\''
if wrap_all_strings:
quoting = csv.QUOTE_NONNUMERIC
else:
quoting = csv.QUOTE_MINIMAL
if 'sqlathanor' in csv.list_dialects():
csv.unregister_dialect('sqlathanor')
csv.register_dialect('sqlathanor',
delimiter=delimiter,
doublequote=double_wrapper_character_when_nested,
escapechar=escape_character,
quotechar=wrapper_character,
quoting=quoting,
lineterminator=line_terminator)
csv_column_names = [
x for x in cls.get_csv_column_names(
deserialize=True, serialize=None, config_set=config_set)
]
csv_reader = csv.DictReader([csv_data],
fieldnames=csv_column_names,
dialect='sqlathanor',
restkey=None,
restval=None)
rows = [x for x in csv_reader]
if len(rows) > 1:
raise CSVStructureError('expected 1 row of data, received %s' %
len(csv_reader))
elif len(rows) == 0:
data = dict_()
for column_name in csv_column_names:
data[column_name] = None
else:
data = rows[0]
if data.get(None, None) is not None:
raise CSVStructureError('expected %s fields, found %s' %
(len(csv_column_names), len(data.keys())))
deserialized_data = dict_()
for key in data:
if data[key] == null_text:
deserialized_data[key] = None
continue
attribute_name = cls._get_attribute_name(key)
deserialized_value = cls._get_deserialized_value(
data[key], 'csv', key, config_set=config_set)
deserialized_data[attribute_name] = deserialized_value
csv.unregister_dialect('sqlathanor')
return deserialized_data
def _parse_dict(cls,
input_data,
format,
error_on_extra_keys=True,
drop_extra_keys=False,
config_set=None):
"""Generate a processed :class:`dict <python:dict>` object from
in-bound :class:`dict <python:dict>` data.
:param input_data: An inbound :class:`dict <python:dict>` object which
can be processed for de-serialization.
:type input_data: :class:`dict <python:dict>`
:param format: The format from which ``input_data`` was received. Accepts:
``'csv'``, ``'json'``, ``'yaml'``, and ``'dict'``.
:type format: :class:`str <python:str>`
:param error_on_extra_keys: If ``True``, will raise an error if an
unrecognized key is found in ``dict_data``. If ``False``, will
either drop or include the extra key in the result, as configured in
the ``drop_extra_keys`` parameter. Defaults to ``True``.
:type error_on_extra_keys: :class:`bool <python:bool>`
:param drop_extra_keys: If ``True``, will omit unrecognized top-level keys
from the resulting :class:`dict <python:dict>`. If ``False``, will
include unrecognized keys or raise an error based on the configuration of
the ``error_on_extra_keys`` parameter. Defaults to ``False``.
:type drop_extra_keys: :class:`bool <python:bool>`
:param config_set: If not :obj:`None <python:None>`, the named configuration set
to use when processing the input. Defaults to :obj:`None <python:None>`.
:type config_set: :class:`str <python:str>` / :obj:`None <python:None>`
:returns: A processed :class:`dict <python:dict>` object that has had
:term:`deserializer functions` applied to it.
:rtype: :class:`dict <python:dict>`
:raises ExtraKeyError: if ``error_on_extra_keys`` is ``True`` and
``input_data`` contains top-level keys that are not recognized as
attributes for the instance model.
:raises DeserializationError: if ``input_data`` is
not a :class:`dict <python:dict>` or JSON object serializable to a
:class:`dict <python:dict>` or if ``input_data`` is empty.
:raises InvalidFormatError: if ``format`` is not a supported value
"""
if format not in ['csv', 'json', 'yaml', 'dict']:
raise InvalidFormatError("format '%s' not supported" % format)
try:
input_data = validators.dict(input_data,
allow_empty=True,
json_serializer=json)
except ValueError:
raise DeserializationError('input_data is not a dict')
if not input_data or len(input_data.keys()) == 0:
raise DeserializationError("input_data is empty")
dict_object = dict_()
if format == 'csv':
attribute_getter = cls.get_csv_serialization_config
elif format == 'json':
attribute_getter = cls.get_json_serialization_config
elif format == 'yaml':
attribute_getter = cls.get_yaml_serialization_config
elif format == 'dict':
attribute_getter = cls.get_dict_serialization_config
attributes = [
x for x in attribute_getter(
deserialize=True, serialize=None, config_set=config_set)
if hasattr(cls, x.name)
]
if not attributes:
raise DeserializableAttributeError(
"'%s' has no '%s' de-serializable attributes" %
(type(cls.__name__), format))
attribute_names = [x.display_name or x.name for x in attributes]
extra_keys = [x for x in input_data.keys() if x not in attribute_names]
if extra_keys and error_on_extra_keys:
raise ExtraKeyError("input data had extra keys: %s" % extra_keys)
for attribute in attributes:
key = attribute.display_name or attribute.name
try:
value = input_data.pop(key)
except KeyError:
continue
value = cls._get_deserialized_value(
value,
format,
key,
error_on_extra_keys=error_on_extra_keys,
drop_extra_keys=drop_extra_keys,
config_set=config_set)
dict_object[attribute.name] = value
if input_data and not drop_extra_keys:
for key in input_data:
dict_object[key] = input_data[key]
return dict_object