def parse_json(input_data, deserialize_function=None, **kwargs): """De-serialize JSON data into a Python :class:`dict <python:dict>` object. :param input_data: The JSON data to de-serialize. :type input_data: :class:`str <python:str>` :param deserialize_function: Optionally override the default JSON deserializer. Defaults to :obj:`None <python:None>`, which calls the default :ref:`simplejson.loads() <simplejson:simplejson.loads>` function from the `simplejson <https://github.com/simplejson/simplejson>`_ library. .. note:: Use the ``deserialize_function`` parameter to override the default YAML deserializer. A valid ``deserialize_function`` is expected to accept a single :class:`str <python:str>` and return a :class:`dict <python:dict>`, similar to :ref:`simplejson.loads() <simplejson:simplejson.loads>` If you wish to pass additional arguments to your ``deserialize_function`` pass them as keyword arguments (in ``kwargs``). :type deserialize_function: callable / :obj:`None <python:None>` :param kwargs: Optional keyword parameters that are passed to the JSON deserializer function. By default, these are options which are passed to :ref:`simplejson.loads() <simplejson:simplejson.loads>`. :type kwargs: keyword arguments :returns: A :class:`dict <python:dict>` representation of ``input_data``. :rtype: :class:`dict <python:dict>` """ is_file = False if checkers.is_file(input_data): is_file = True if deserialize_function is None and not is_file: deserialize_function = json.loads elif deserialize_function is None and is_file: deserialize_function = json.load else: if checkers.is_callable(deserialize_function) is False: raise ValueError('deserialize_function (%s) is not callable' % deserialize_function) if not input_data: raise DeserializationError('input_data is empty') if not is_file: try: input_data = validators.string(input_data, allow_empty=False) except ValueError: raise DeserializationError('input_data is not a valid string') from_json = deserialize_function(input_data, **kwargs) else: with open(input_data, 'r') as input_file: from_json = deserialize_function(input_file, **kwargs) return from_json
def parse_yaml(input_data, deserialize_function=None, **kwargs): """De-serialize YAML data into a Python :class:`dict <python:dict>` object. :param input_data: The YAML data to de-serialize. :type input_data: :class:`str <python:str>` :param deserialize_function: Optionally override the default YAML deserializer. Defaults to :obj:`None <python:None>`, which calls the default ``yaml.safe_load()`` function from the `PyYAML <https://github.com/yaml/pyyaml>`_ library. .. note:: Use the ``deserialize_function`` parameter to override the default YAML deserializer. A valid ``deserialize_function`` is expected to accept a single :class:`str <python:str>` and return a :class:`dict <python:dict>`, similar to ``yaml.safe_load()``. If you wish to pass additional arguments to your ``deserialize_function`` pass them as keyword arguments (in ``kwargs``). :type deserialize_function: callable / :obj:`None <python:None>` :param kwargs: Optional keyword parameters that are passed to the YAML deserializer function. By default, these are options which are passed to ``yaml.safe_load()``. :type kwargs: keyword arguments :returns: A :class:`dict <python:dict>` representation of ``input_data``. :rtype: :class:`dict <python:dict>` """ if deserialize_function is None: deserialize_function = yaml.safe_load else: if checkers.is_callable(deserialize_function) is False: raise ValueError('deserialize_function (%s) is not callable' % deserialize_function) if not input_data: raise DeserializationError('input_data is empty') try: input_data = validators.string(input_data, allow_empty=False) except ValueError: raise DeserializationError('input_data is not a valid string') from_yaml = yaml.safe_load(input_data, **kwargs) return from_yaml
def parse_csv(input_data, delimiter='|', wrap_all_strings=False, null_text='None', wrapper_character="'", double_wrapper_character_when_nested=False, escape_character="\\", line_terminator='\r\n'): """De-serialize CSV data into a Python :class:`dict <python:dict>` object. .. versionadded:: 0.3.0 .. tip:: Unwrapped empty column values are automatically interpreted as null (:obj:`None <python:None>`). :param input_data: The CSV data to de-serialize. Should include column headers and at least **one** row of data. Will ignore any rows of data beyond the first row. :type input_data: :class:`str <python:str>` :param delimiter: The delimiter used between columns. Defaults to ``|``. :type delimiter: :class:`str <python:str>` :param wrapper_character: The string used to wrap string values when wrapping is applied. Defaults to ``'``. :type wrapper_character: :class:`str <python:str>` :param null_text: The string used to indicate an empty value if empty values are wrapped. Defaults to `None`. :type null_text: :class:`str <python:str>` :returns: A :class:`dict <python:dict>` representation of the CSV record. :rtype: :class:`dict <python:dict>` :raises DeserializationError: if ``input_data`` is not a valid :class:`str <python:str>` :raises CSVStructureError: if there are less than 2 (two) rows in ``input_data`` or if column headers are not valid Python variable names """ use_file = False if not checkers.is_file(input_data) and not checkers.is_iterable( input_data): try: input_data = validators.string(input_data, allow_empty=False) except (ValueError, TypeError): raise DeserializationError("input_data expects a 'str', received '%s'" \ % type(input_data)) input_data = [input_data] elif checkers.is_file(input_data): use_file = True if not wrapper_character: wrapper_character = '\'' if wrap_all_strings: quoting = csv.QUOTE_NONNUMERIC else: quoting = csv.QUOTE_MINIMAL if 'sqlathanor' in csv.list_dialects(): csv.unregister_dialect('sqlathanor') csv.register_dialect('sqlathanor', delimiter=delimiter, doublequote=double_wrapper_character_when_nested, escapechar=escape_character, quotechar=wrapper_character, quoting=quoting, lineterminator=line_terminator) if not use_file: csv_reader = csv.DictReader(input_data, dialect='sqlathanor', restkey=None, restval=None) rows = [x for x in csv_reader] else: if not is_py2: with open(input_data, 'r', newline='') as input_file: csv_reader = csv.DictReader(input_file, dialect='sqlathanor', restkey=None, restval=None) rows = [x for x in csv_reader] else: with open(input_data, 'r') as input_file: csv_reader = csv.DictReader(input_file, dialect='sqlathanor', restkey=None, restval=None) rows = [x for x in csv_reader] if len(rows) < 1: raise CSVStructureError( 'expected 1 row of data and 1 header row, missing 1') else: data = rows[0] for key in data: try: validators.variable_name(key) except ValueError: raise CSVStructureError( 'column (%s) is not a valid Python variable name' % key) if data[key] == null_text: data[key] = None csv.unregister_dialect('sqlathanor') return data
def _parse_csv(cls, csv_data, delimiter='|', wrap_all_strings=False, null_text='None', wrapper_character="'", double_wrapper_character_when_nested=False, escape_character="\\", line_terminator='\r\n', config_set=None): """Generate a :class:`dict <python:dict>` from a CSV record. .. tip:: Unwrapped empty column values are automatically interpreted as null (:obj:`None <python:None>`). :param csv_data: The CSV record. Should be a single row and should **not** include column headers. :type csv_data: :class:`str <python:str>` :param delimiter: The delimiter used between columns. Defaults to ``|``. :type delimiter: :class:`str <python:str>` :param wrapper_character: The string used to wrap string values when wrapping is applied. Defaults to ``'``. :type wrapper_character: :class:`str <python:str>` :param null_text: The string used to indicate an empty value if empty values are wrapped. Defaults to `None`. :type null_text: :class:`str <python:str>` :param config_set: If not :obj:`None <python:None>`, the named configuration set to use. Defaults to :obj:`None <python:None>`. :type config_set: :class:`str <python:str>` / :obj:`None <python:None>` :returns: A :class:`dict <python:dict>` representation of the CSV record. :rtype: :class:`dict <python:dict>` :raises DeserializationError: if ``csv_data`` is not a valid :class:`str <python:str>` :raises CSVStructureError: if the columns in ``csv_data`` do not match the expected columns returned by :func:`get_csv_column_names() <BaseModel.get_csv_column_names>` :raises ValueDeserializationError: if a value extracted from the CSV failed when executing its :term:`de-serialization function`. """ try: csv_data = validators.string(csv_data, allow_empty=False) except (ValueError, TypeError): raise DeserializationError("csv_data expects a 'str', received '%s'" \ % type(csv_data)) if not wrapper_character: wrapper_character = '\'' if wrap_all_strings: quoting = csv.QUOTE_NONNUMERIC else: quoting = csv.QUOTE_MINIMAL if 'sqlathanor' in csv.list_dialects(): csv.unregister_dialect('sqlathanor') csv.register_dialect('sqlathanor', delimiter=delimiter, doublequote=double_wrapper_character_when_nested, escapechar=escape_character, quotechar=wrapper_character, quoting=quoting, lineterminator=line_terminator) csv_column_names = [ x for x in cls.get_csv_column_names( deserialize=True, serialize=None, config_set=config_set) ] csv_reader = csv.DictReader([csv_data], fieldnames=csv_column_names, dialect='sqlathanor', restkey=None, restval=None) rows = [x for x in csv_reader] if len(rows) > 1: raise CSVStructureError('expected 1 row of data, received %s' % len(csv_reader)) elif len(rows) == 0: data = dict_() for column_name in csv_column_names: data[column_name] = None else: data = rows[0] if data.get(None, None) is not None: raise CSVStructureError('expected %s fields, found %s' % (len(csv_column_names), len(data.keys()))) deserialized_data = dict_() for key in data: if data[key] == null_text: deserialized_data[key] = None continue attribute_name = cls._get_attribute_name(key) deserialized_value = cls._get_deserialized_value( data[key], 'csv', key, config_set=config_set) deserialized_data[attribute_name] = deserialized_value csv.unregister_dialect('sqlathanor') return deserialized_data
def _parse_dict(cls, input_data, format, error_on_extra_keys=True, drop_extra_keys=False, config_set=None): """Generate a processed :class:`dict <python:dict>` object from in-bound :class:`dict <python:dict>` data. :param input_data: An inbound :class:`dict <python:dict>` object which can be processed for de-serialization. :type input_data: :class:`dict <python:dict>` :param format: The format from which ``input_data`` was received. Accepts: ``'csv'``, ``'json'``, ``'yaml'``, and ``'dict'``. :type format: :class:`str <python:str>` :param error_on_extra_keys: If ``True``, will raise an error if an unrecognized key is found in ``dict_data``. If ``False``, will either drop or include the extra key in the result, as configured in the ``drop_extra_keys`` parameter. Defaults to ``True``. :type error_on_extra_keys: :class:`bool <python:bool>` :param drop_extra_keys: If ``True``, will omit unrecognized top-level keys from the resulting :class:`dict <python:dict>`. If ``False``, will include unrecognized keys or raise an error based on the configuration of the ``error_on_extra_keys`` parameter. Defaults to ``False``. :type drop_extra_keys: :class:`bool <python:bool>` :param config_set: If not :obj:`None <python:None>`, the named configuration set to use when processing the input. Defaults to :obj:`None <python:None>`. :type config_set: :class:`str <python:str>` / :obj:`None <python:None>` :returns: A processed :class:`dict <python:dict>` object that has had :term:`deserializer functions` applied to it. :rtype: :class:`dict <python:dict>` :raises ExtraKeyError: if ``error_on_extra_keys`` is ``True`` and ``input_data`` contains top-level keys that are not recognized as attributes for the instance model. :raises DeserializationError: if ``input_data`` is not a :class:`dict <python:dict>` or JSON object serializable to a :class:`dict <python:dict>` or if ``input_data`` is empty. :raises InvalidFormatError: if ``format`` is not a supported value """ if format not in ['csv', 'json', 'yaml', 'dict']: raise InvalidFormatError("format '%s' not supported" % format) try: input_data = validators.dict(input_data, allow_empty=True, json_serializer=json) except ValueError: raise DeserializationError('input_data is not a dict') if not input_data or len(input_data.keys()) == 0: raise DeserializationError("input_data is empty") dict_object = dict_() if format == 'csv': attribute_getter = cls.get_csv_serialization_config elif format == 'json': attribute_getter = cls.get_json_serialization_config elif format == 'yaml': attribute_getter = cls.get_yaml_serialization_config elif format == 'dict': attribute_getter = cls.get_dict_serialization_config attributes = [ x for x in attribute_getter( deserialize=True, serialize=None, config_set=config_set) if hasattr(cls, x.name) ] if not attributes: raise DeserializableAttributeError( "'%s' has no '%s' de-serializable attributes" % (type(cls.__name__), format)) attribute_names = [x.display_name or x.name for x in attributes] extra_keys = [x for x in input_data.keys() if x not in attribute_names] if extra_keys and error_on_extra_keys: raise ExtraKeyError("input data had extra keys: %s" % extra_keys) for attribute in attributes: key = attribute.display_name or attribute.name try: value = input_data.pop(key) except KeyError: continue value = cls._get_deserialized_value( value, format, key, error_on_extra_keys=error_on_extra_keys, drop_extra_keys=drop_extra_keys, config_set=config_set) dict_object[attribute.name] = value if input_data and not drop_extra_keys: for key in input_data: dict_object[key] = input_data[key] return dict_object