def _parse_csv(cls,
csv_data,
delimiter='|',
wrap_all_strings=False,
null_text='None',
wrapper_character="'",
double_wrapper_character_when_nested=False,
escape_character="\\",
line_terminator='\r\n',
config_set=None):
"""Generate a :class:`dict <python:dict>` from a CSV record.
.. tip::
Unwrapped empty column values are automatically interpreted as null
(:obj:`None <python:None>`).
:param csv_data: The CSV record. Should be a single row and should **not**
include column headers.
:type csv_data: :class:`str <python:str>`
:param delimiter: The delimiter used between columns. Defaults to ``|``.
:type delimiter: :class:`str <python:str>`
:param wrapper_character: The string used to wrap string values when
wrapping is applied. Defaults to ``'``.
:type wrapper_character: :class:`str <python:str>`
:param null_text: The string used to indicate an empty value if empty
values are wrapped. Defaults to `None`.
:type null_text: :class:`str <python:str>`
:param config_set: If not :obj:`None <python:None>`, the named configuration set
to use. Defaults to :obj:`None <python:None>`.
:type config_set: :class:`str <python:str>` / :obj:`None <python:None>`
:returns: A :class:`dict <python:dict>` representation of the CSV record.
:rtype: :class:`dict <python:dict>`
:raises DeserializationError: if ``csv_data`` is not a valid
:class:`str <python:str>`
:raises CSVStructureError: if the columns in ``csv_data`` do not match
the expected columns returned by
:func:`get_csv_column_names() <BaseModel.get_csv_column_names>`
:raises ValueDeserializationError: if a value extracted from the CSV
failed when executing its :term:`de-serialization function`.
"""
try:
csv_data = validators.string(csv_data, allow_empty=False)
except (ValueError, TypeError):
raise DeserializationError("csv_data expects a 'str', received '%s'" \
% type(csv_data))
if not wrapper_character:
wrapper_character = '\''
if wrap_all_strings:
quoting = csv.QUOTE_NONNUMERIC
else:
quoting = csv.QUOTE_MINIMAL
if 'sqlathanor' in csv.list_dialects():
csv.unregister_dialect('sqlathanor')
csv.register_dialect('sqlathanor',
delimiter=delimiter,
doublequote=double_wrapper_character_when_nested,
escapechar=escape_character,
quotechar=wrapper_character,
quoting=quoting,
lineterminator=line_terminator)
csv_column_names = [
x for x in cls.get_csv_column_names(
deserialize=True, serialize=None, config_set=config_set)
]
csv_reader = csv.DictReader([csv_data],
fieldnames=csv_column_names,
dialect='sqlathanor',
restkey=None,
restval=None)
rows = [x for x in csv_reader]
if len(rows) > 1:
raise CSVStructureError('expected 1 row of data, received %s' %
len(csv_reader))
elif len(rows) == 0:
data = dict_()
for column_name in csv_column_names:
data[column_name] = None
else:
data = rows[0]
if data.get(None, None) is not None:
raise CSVStructureError('expected %s fields, found %s' %
(len(csv_column_names), len(data.keys())))
deserialized_data = dict_()
for key in data:
if data[key] == null_text:
deserialized_data[key] = None
continue
attribute_name = cls._get_attribute_name(key)
deserialized_value = cls._get_deserialized_value(
data[key], 'csv', key, config_set=config_set)
deserialized_data[attribute_name] = deserialized_value
csv.unregister_dialect('sqlathanor')
return deserialized_data
def _parse_dict(cls,
input_data,
format,
error_on_extra_keys=True,
drop_extra_keys=False,
config_set=None):
"""Generate a processed :class:`dict <python:dict>` object from
in-bound :class:`dict <python:dict>` data.
:param input_data: An inbound :class:`dict <python:dict>` object which
can be processed for de-serialization.
:type input_data: :class:`dict <python:dict>`
:param format: The format from which ``input_data`` was received. Accepts:
``'csv'``, ``'json'``, ``'yaml'``, and ``'dict'``.
:type format: :class:`str <python:str>`
:param error_on_extra_keys: If ``True``, will raise an error if an
unrecognized key is found in ``dict_data``. If ``False``, will
either drop or include the extra key in the result, as configured in
the ``drop_extra_keys`` parameter. Defaults to ``True``.
:type error_on_extra_keys: :class:`bool <python:bool>`
:param drop_extra_keys: If ``True``, will omit unrecognized top-level keys
from the resulting :class:`dict <python:dict>`. If ``False``, will
include unrecognized keys or raise an error based on the configuration of
the ``error_on_extra_keys`` parameter. Defaults to ``False``.
:type drop_extra_keys: :class:`bool <python:bool>`
:param config_set: If not :obj:`None <python:None>`, the named configuration set
to use when processing the input. Defaults to :obj:`None <python:None>`.
:type config_set: :class:`str <python:str>` / :obj:`None <python:None>`
:returns: A processed :class:`dict <python:dict>` object that has had
:term:`deserializer functions` applied to it.
:rtype: :class:`dict <python:dict>`
:raises ExtraKeyError: if ``error_on_extra_keys`` is ``True`` and
``input_data`` contains top-level keys that are not recognized as
attributes for the instance model.
:raises DeserializationError: if ``input_data`` is
not a :class:`dict <python:dict>` or JSON object serializable to a
:class:`dict <python:dict>` or if ``input_data`` is empty.
:raises InvalidFormatError: if ``format`` is not a supported value
"""
if format not in ['csv', 'json', 'yaml', 'dict']:
raise InvalidFormatError("format '%s' not supported" % format)
try:
input_data = validators.dict(input_data,
allow_empty=True,
json_serializer=json)
except ValueError:
raise DeserializationError('input_data is not a dict')
if not input_data or len(input_data.keys()) == 0:
raise DeserializationError("input_data is empty")
dict_object = dict_()
if format == 'csv':
attribute_getter = cls.get_csv_serialization_config
elif format == 'json':
attribute_getter = cls.get_json_serialization_config
elif format == 'yaml':
attribute_getter = cls.get_yaml_serialization_config
elif format == 'dict':
attribute_getter = cls.get_dict_serialization_config
attributes = [
x for x in attribute_getter(
deserialize=True, serialize=None, config_set=config_set)
if hasattr(cls, x.name)
]
if not attributes:
raise DeserializableAttributeError(
"'%s' has no '%s' de-serializable attributes" %
(type(cls.__name__), format))
attribute_names = [x.display_name or x.name for x in attributes]
extra_keys = [x for x in input_data.keys() if x not in attribute_names]
if extra_keys and error_on_extra_keys:
raise ExtraKeyError("input data had extra keys: %s" % extra_keys)
for attribute in attributes:
key = attribute.display_name or attribute.name
try:
value = input_data.pop(key)
except KeyError:
continue
value = cls._get_deserialized_value(
value,
format,
key,
error_on_extra_keys=error_on_extra_keys,
drop_extra_keys=drop_extra_keys,
config_set=config_set)
dict_object[attribute.name] = value
if input_data and not drop_extra_keys:
for key in input_data:
dict_object[key] = input_data[key]
return dict_object
def _get_attribute_csv_data(self,
attributes,
is_dumping=False,
delimiter='|',
wrap_all_strings=False,
null_text='None',
wrapper_character="'",
double_wrapper_character_when_nested=False,
escape_character="\\",
line_terminator='\r\n',
config_set=None):
r"""Return the CSV representation of ``attributes`` extracted from the
model instance (record).
:param attributes: Names of :term:`model attributes <model attribute>` to
include in the CSV output.
:type attributes: :class:`list <python:list>` of :class:`str <python:str>`
:param is_dumping: If ``True``, then allow
:exc:`UnsupportedSerializationError <sqlathanor.errors.UnsupportedSerializationError>`.
Defaults to ``False``.
:type is_dumping: :class:`bool <python:bool>`
:param delimiter: The delimiter used between columns. Defaults to ``|``.
:type delimiter: :class:`str <python:str>`
:param wrap_all_strings: If ``True``, wraps any string data in the
``wrapper_character``. If ``None``, only wraps string data if it contains
the ``delimiter``. Defaults to ``False``.
:type wrap_all_strings: :class:`bool <python:bool>`
:param null_text: The text value to use in place of empty values. Only
applies if ``wrap_empty_values`` is ``True``. Defaults to ``'None'``.
:type null_text: :class:`str <python:str>`
:param wrapper_character: The string used to wrap string values when
wrapping is necessary. Defaults to ``'``.
:type wrapper_character: :class:`str <python:str>`
:param double_wrapper_character_when_nested: If ``True``, will double the
``wrapper_character`` when it is found inside a column value. If ``False``,
will precede the ``wrapper_character`` by the ``escape_character`` when
it is found inside a column value. Defaults to ``False``.
:type double_wrapper_character_when_nested: :class:`bool <python:bool>`
:param escape_character: The character to use when escaping nested wrapper
characters. Defaults to ``\``.
:type escape_character: :class:`str <python:str>`
:param line_terminator: The character used to mark the end of a line.
Defaults to ``\r\n``.
:type line_terminator: :class:`str <python:str>`
:param config_set: If not :obj:`None <python:None>`, the named configuration set
to use. Defaults to :obj:`None <python:None>`.
:type config_set: :class:`str <python:str>` / :obj:`None <python:None>`
:returns: Data from the object in CSV format ending in ``line_terminator``.
:rtype: :class:`str <python:str>`
"""
if not wrapper_character:
wrapper_character = '\''
if not attributes:
raise SerializableAttributeError("attributes cannot be empty")
if wrap_all_strings:
quoting = csv.QUOTE_NONNUMERIC
else:
quoting = csv.QUOTE_MINIMAL
if 'sqlathanor' in csv.list_dialects():
csv.unregister_dialect('sqlathanor')
csv.register_dialect('sqlathanor',
delimiter=delimiter,
doublequote=double_wrapper_character_when_nested,
escapechar=escape_character,
quotechar=wrapper_character,
quoting=quoting,
lineterminator=line_terminator)
data = []
for item in attributes:
try:
value = self._get_serialized_value(format='csv',
attribute=item,
config_set=config_set)
except UnsupportedSerializationError as error:
if is_dumping:
value = getattr(self, item)
else:
raise error
data.append(value)
for index, item in enumerate(data):
if item == '' or item is None or item == 'None':
data[index] = null_text
elif not checkers.is_string(item) and not checkers.is_numeric(
item):
data[index] = str(item)
data_dict = dict_()
for index, column_name in enumerate(attributes):
data_dict[column_name] = data[index]
output = StringIO()
csv_writer = csv.DictWriter(output,
fieldnames=attributes,
dialect='sqlathanor')
csv_writer.writerow(data_dict)
data_row = output.getvalue()
output.close()
csv.unregister_dialect('sqlathanor')
return data_row
def _to_dict(self,
format,
max_nesting=0,
current_nesting=0,
is_dumping=False,
config_set=None):
"""Return a :class:`dict <python:dict>` representation of the object.
.. warning::
This method is an **intermediate** step that is used to produce the
contents for certain public JSON, YAML, and :class:`dict <python:dict>`
serialization methods. It should not be called directly.
:param format: The format to which the :class:`dict <python:dict>` will
ultimately be serialized. Accepts: ``'csv'``, ``'json'``, ``'yaml'``, and
``'dict'``.
:type format: :class:`str <python:str>`
:param max_nesting: The maximum number of levels that the resulting
:class:`dict <python:dict>` object can be nested. If set to ``0``, will
not nest other serializable objects. Defaults to ``0``.
:type max_nesting: :class:`int <python:int>`
:param current_nesting: The current nesting level at which the
:class:`dict <python:dict>` representation will reside. Defaults to ``0``.
:type current_nesting: :class:`int <python:int>`
:param is_dumping: If ``True``, retrieves all attributes except callables,
utilities, and specials (``__<name>``). If ``False``, only retrieves
those that have JSON serialization enabled. Defaults to ``False``.
:type is_dumping: :class:`bool <python:bool>`
:param config_set: If not :obj:`None <python:None>`, the named configuration set
to use when processing the input. Defaults to :obj:`None <python:None>`.
:type config_set: :class:`str <python:str>` / :obj:`None <python:None>`
:returns: A :class:`dict <python:dict>` representation of the object.
:rtype: :class:`dict <python:dict>`
:raises InvalidFormatError: if ``format`` is not recognized
:raises SerializableAttributeError: if attributes is empty
:raises UnsupportedSerializationError: if unable to serialize a value
:raises MaximumNestingExceededError: if ``current_nesting`` is greater
than ``max_nesting``
:raises MaximumNestingExceededWarning: if an attribute requires nesting
beyond ``max_nesting``
"""
# pylint: disable=too-many-branches
next_nesting = current_nesting + 1
if format not in ['csv', 'json', 'yaml', 'dict']:
raise InvalidFormatError("format '%s' not supported" % format)
if current_nesting > max_nesting:
raise MaximumNestingExceededError(
'current nesting level (%s) exceeds maximum %s' %
(current_nesting, max_nesting))
dict_object = dict_()
if format == 'csv':
attribute_getter = self.get_csv_serialization_config
elif format == 'json':
attribute_getter = self.get_json_serialization_config
elif format == 'yaml':
attribute_getter = self.get_yaml_serialization_config
elif format == 'dict':
attribute_getter = self.get_dict_serialization_config
if not is_dumping:
attributes = [
x for x in attribute_getter(
deserialize=None, serialize=True, config_set=config_set)
if hasattr(self, x.name)
]
else:
attribute_names = [
x for x in get_attribute_names(self,
include_callable=False,
include_nested=False,
include_private=True,
include_special=False,
include_utilities=False)
]
attributes = []
for item in attribute_names:
attribute_config = self.get_attribute_serialization_config(
item, config_set=config_set)
if attribute_config is not None:
on_serialize_function = attribute_config.on_serialize.get(
format, None)
else:
on_serialize_function = None
attribute = AttributeConfiguration(
name=item,
supports_json=True,
supports_yaml=True,
supports_dict=True,
on_serialize=on_serialize_function)
attributes.append(attribute)
if not attributes:
raise SerializableAttributeError(
"'%s' has no '%s' serializable attributes" %
(type(self.__class__), format))
for attribute in attributes:
item = getattr(self, attribute.name, None)
if hasattr(item, '_to_dict'):
try:
value = item._to_dict(
format, # pylint: disable=protected-access
max_nesting=max_nesting,
current_nesting=next_nesting,
is_dumping=is_dumping,
config_set=config_set)
except MaximumNestingExceededError:
warnings.warn(
"skipping key '%s' because maximum nesting has been exceeded" \
% attribute.name,
MaximumNestingExceededWarning
)
continue
else:
if attribute.on_serialize[format]:
on_serialize_function = attribute.on_serialize[format]
item = on_serialize_function(item)
if checkers.is_iterable(item,
forbid_literals=(str, bytes, dict)):
try:
value = iterable__to_dict(item,
format,
max_nesting=max_nesting,
current_nesting=next_nesting,
is_dumping=is_dumping,
config_set=config_set)
except MaximumNestingExceededError:
warnings.warn(
"skipping key '%s' because maximum nesting has been exceeded" \
% attribute.name,
MaximumNestingExceededWarning
)
continue
except NotAnIterableError:
try:
value = self._get_serialized_value(
format, attribute.name, config_set=config_set)
except UnsupportedSerializationError as error:
if is_dumping:
value = getattr(self, attribute.name)
else:
raise error
else:
try:
value = self._get_serialized_value(
format, attribute.name, config_set=config_set)
except UnsupportedSerializationError as error:
if is_dumping:
value = getattr(self, attribute.name)
else:
raise error
serialized_key = attribute.display_name or attribute.name
dict_object[str(serialized_key)] = value
return dict_object