def test_smart_items(self):
d = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
d['c.d'] = [{'e': 4}, {'f': 5}]
self.assertTrue(d.items(), [('a', [{'b': 1}, {'b': 2}, {'b': 3}]),
('c', {'d': [{'e': 4}, {'f': 5}]}),
('foo', {'a': 'world', 'b': 'hello'})])
def test_smart_itervalues(self):
d = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['c.d'] = [{'e': 4}, {'f': 5}]
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
iterator = d.itervalues()
self.assertEqual(iterator.next(), [{'b': 1}, {'b': 2}, {'b': 3}])
self.assertEqual(iterator.next(), {'d': [{'e': 4}, {'f': 5}]})
self.assertEqual(iterator.next(), {'a': 'world', 'b': 'hello'})
self.assertRaises(StopIteration, iterator.next)
def test_smart_itervalues(self):
d = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['c.d'] = [{'e': 4}, {'f': 5}]
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
iterator = d.itervalues()
self.assertEqual(iterator.next(), [{'b': 1}, {'b': 2}, {'b': 3}])
self.assertEqual(iterator.next(), {'d': [{'e': 4}, {'f': 5}]})
self.assertEqual(iterator.next(), {'a': 'world', 'b': 'hello'})
self.assertRaises(StopIteration, iterator.next)
def test_smart__contains(self):
d = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
self.assertFalse('.' in d)
self.assertFalse('[' in d)
def test_smart_has_key(self):
d = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['c.d'] = [{'e': 4}, {'f': 5}]
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
self.assertTrue('c' in d)
self.assertFalse('v' in d)
self.assertTrue('c.d' in d)
self.assertTrue('foo.b' in d)
def test_smart_update(self):
d = SmartDict()
d2 = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['c.d'] = [{'e': 4}, {'f': 5}]
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
d2['qwerty'] = {'t': 'u', 'i': 'o'}
d2['qwe.rty'] = [{'n': 34}, {'x': 3}]
d.update(d2)
d3 = SmartDict({'a': [{'b': 1}, {'b': 2}, {'b': 3}],
'c': {'d': [{'e': 4}, {'f': 5}]},
'foo': {'a': 'world', 'b': 'hello'},
'qwe': {'rty': [{'n': 34}, {'x': 3}]},
'qwerty': {'i': 'o', 't': 'u'}})
self.assertTrue(d == d3)
def test_smart_iter(self):
d = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
iterator = iter(d)
self.assertEqual(iterator.next(), 'a')
self.assertEqual(iterator.next(), 'foo')
self.assertRaises(StopIteration, iterator.next)
def test_smart_items(self):
d = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
d['c.d'] = [{'e': 4}, {'f': 5}]
self.assertTrue(d.items(), [('a', [{
'b': 1
}, {
'b': 2
}, {
'b': 3
}]), ('c', {
'd': [{
'e': 4
}, {
'f': 5
}]
}), ('foo', {
'a': 'world',
'b': 'hello'
})])
def test_smart_repr(self):
d = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['c.d'] = [{'e': 4}, {'f': 5}]
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
self.assertEqual(
repr(d), "{'a': [{'b': 1}, {'b': 2}, {'b': 3}], " +
"'c': {'d': [{'e': 4}, {'f': 5}]}, " +
"'foo': {'a': 'world', 'b': 'hello'}}")
del d['c']
self.assertEqual(
repr(d), "{'a': [{'b': 1}, {'b': 2}, {'b': 3}], " +
"'foo': {'a': 'world', 'b': 'hello'}}")
def test_smart_update(self):
d = SmartDict()
d2 = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['c.d'] = [{'e': 4}, {'f': 5}]
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
d2['qwerty'] = {'t': 'u', 'i': 'o'}
d2['qwe.rty'] = [{'n': 34}, {'x': 3}]
d.update(d2)
d3 = SmartDict({
'a': [{
'b': 1
}, {
'b': 2
}, {
'b': 3
}],
'c': {
'd': [{
'e': 4
}, {
'f': 5
}]
},
'foo': {
'a': 'world',
'b': 'hello'
},
'qwe': {
'rty': [{
'n': 34
}, {
'x': 3
}]
},
'qwerty': {
'i': 'o',
't': 'u'
}
})
self.assertTrue(d == d3)
def test_smart_dict(self):
d = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
self.assertEqual(d.keys(), ['a', 'foo'])
self.assertTrue('foo.a' in d)
del d['foo']
self.assertEqual(d.keys(), ['a'])
self.assertEqual(d['a'], [{'b': 1}, {'b': 2}, {'b': 3}])
self.assertEqual(d['a[0]'], {'b': 1})
self.assertEqual(d['a.b'], [1, 2, 3])
self.assertEqual(d['a[1:]'], [{'b': 2}, {'b': 3}])
d.set('a', {'b': 4}, extend=True)
self.assertEqual(d['a'], [{'b': 1}, {'b': 2}, {'b': 3}, {'b': 4}])
d.set('a', [{'b': 1}, {'b': 2}, {'b': 3}], extend=False)
self.assertEqual(d['a'], [{'b': 1}, {'b': 2}, {'b': 3}])
self.assertEqual(d.get('does not exists'), None)
d = SmartDict()
d.set('a.b.c[n]', 'foo', True)
self.assertEqual(d['a.b.c'], ['foo'])
d.set('a.b.c[n]', 'bar', True)
self.assertEqual(d['a.b.c'], ['foo', 'bar'])
d.set('a.b.c', ['foo'], False)
self.assertEqual(d['a.b.c'], ['foo'])
def __init__(self,
json=None,
set_default_values=False,
process_model_info=False,
**kwargs):
"""If no JSON, a new structure will be created.
Typically the JSON structure needs a few mandatory fields and subfields
that are created here, the final JSON should look like this::
{
'__meta_metadata': {
'__additional_info__': {...},
'__model_info__': {'model_names': [...], ....},
'__aliases__': {...},
'__errors__': [...],
'__continuable_errors__': []
}
}
The content of `__additional_info__` is, usually, the content of
kwargs.
This object maintains two different dictionaries, one with the pure
JSON representation, `_dict` and a second one with the BSON
representation of the former. This behavior helps whenever dealing with
JSON databases like PostgreSQL.
The content of the BSON dictionary is typically the same as in the JSON
one, only if the `json` section is described in the definition of the
field the content might change, a good example of this behavior are the
date fields.
See :class:`~.jsonext.parsers.json_extra_parser:JsonExtraParser`.
:param json: JSON to build the dictionary.
:param set_defaul_values: If `True` default values will be set.
:param process_model_info: If `True` all model info will be parsed.
:param kwargs: Everything that would be useful inside
`__additional_info__`. Although it could be used also for:
* `model=['model1', 'model2']`, in this case this key will be
deleted from `kwargs` and used as model names inside
`__model_info__`.
"""
super(SmartJson, self).__init__(json)
self._dict_bson = SmartDict()
if not json or '__meta_metadata__' not in json:
model_names = kwargs.get('model', [
'__default__',
])
if 'model' in kwargs:
del kwargs['model']
if isinstance(model_names, six.string_types):
model_names = [
model_names,
]
self._dict['__meta_metadata__'] = dict()
self._dict['__meta_metadata__']['__additional_info__'] = kwargs
self._dict['__meta_metadata__']['__model_info__'] = \
dict(names=model_names)
self._dict['__meta_metadata__']['__aliases__'] = dict()
self._dict['__meta_metadata__']['__errors__'] = list()
self._dict['__meta_metadata__']['__continuable_errors__'] = list()
if process_model_info:
Reader.process_model_info(self)
if set_default_values:
self.set_default_values()
class SmartJson(SmartDict):
"""Base class for Json structures."""
def __init__(self,
json=None,
set_default_values=False,
process_model_info=False,
**kwargs):
"""If no JSON, a new structure will be created.
Typically the JSON structure needs a few mandatory fields and subfields
that are created here, the final JSON should look like this::
{
'__meta_metadata': {
'__additional_info__': {...},
'__model_info__': {'model_names': [...], ....},
'__aliases__': {...},
'__errors__': [...],
'__continuable_errors__': []
}
}
The content of `__additional_info__` is, usually, the content of
kwargs.
This object maintains two different dictionaries, one with the pure
JSON representation, `_dict` and a second one with the BSON
representation of the former. This behavior helps whenever dealing with
JSON databases like PostgreSQL.
The content of the BSON dictionary is typically the same as in the JSON
one, only if the `json` section is described in the definition of the
field the content might change, a good example of this behavior are the
date fields.
See :class:`~.jsonext.parsers.json_extra_parser:JsonExtraParser`.
:param json: JSON to build the dictionary.
:param set_defaul_values: If `True` default values will be set.
:param process_model_info: If `True` all model info will be parsed.
:param kwargs: Everything that would be useful inside
`__additional_info__`. Although it could be used also for:
* `model=['model1', 'model2']`, in this case this key will be
deleted from `kwargs` and used as model names inside
`__model_info__`.
"""
super(SmartJson, self).__init__(json)
self._dict_bson = SmartDict()
if not json or '__meta_metadata__' not in json:
model_names = kwargs.get('model', [
'__default__',
])
if 'model' in kwargs:
del kwargs['model']
if isinstance(model_names, six.string_types):
model_names = [
model_names,
]
self._dict['__meta_metadata__'] = dict()
self._dict['__meta_metadata__']['__additional_info__'] = kwargs
self._dict['__meta_metadata__']['__model_info__'] = \
dict(names=model_names)
self._dict['__meta_metadata__']['__aliases__'] = dict()
self._dict['__meta_metadata__']['__errors__'] = list()
self._dict['__meta_metadata__']['__continuable_errors__'] = list()
if process_model_info:
Reader.process_model_info(self)
if set_default_values:
self.set_default_values()
@property
def additional_info(self):
"""Shortcut to `__meta_metadata__.__additional_info__`."""
return DotableDict(self['__meta_metadata__']['__additional_info__'])
@property
def errors(self):
"""Shortcut to `__meta_metadata__.__errors__`."""
return self._dict['__meta_metadata__']['__errors__']
@property
def continuable_errors(self):
"""Shortcut to `__meta_metadata__.__continuable_errors__`."""
return self._dict['__meta_metadata__']['__continuable_errors__']
@property
def meta_metadata(self):
"""Shortcut to `__meta_metadata__`."""
return DotableDict(self._dict['__meta_metadata__'])
@property
def model_info(self):
"""Shortcut to `__meta_metadata__.__model_info__`."""
return DotableDict(self._dict['__meta_metadata__']['__model_info__'])
def __getitem__(self, key, reset=False, **kwargs):
"""Like in `dict.__getitem__`.
First it tries to load the value from the value from the BSON object,
if it fails, or reset is set to `True`, looks into the JSON object for
the `key`, applies all the extensions and decorators and return the
value that is stored in the BSON object as a cached version.
If the key is not found inside the dictionary, it tries before raising
`KeyError` to figure out if it is dealing with an alias.
:param key:
:param reset: If the key corresponds to a field calculated on the fly
the value will be calculated again.
:param kwargs: Typically used to set:
* `action`, Whether we are performing a `set` or a `get`.
*`exclude`, from the list of extensions and decorators excludes the
ones that are not required.
:return: Like in `dict.__getitem__`
"""
try:
value = self._dict_bson[key]
if value and not reset:
return value
except KeyError:
# Try to find the key inside the json dict and load it
pass
main_key = SmartDict.main_key_pattern.sub('', key)
if main_key == '__meta_metadata__':
return super(SmartJson, self).__getitem__(key)
elif main_key in self._dict:
if main_key not in self.meta_metadata:
Reader.set(self, main_key)
action = kwargs.get('action', 'get')
exclude = kwargs.get('exclude', [])
if 'extensions' not in exclude:
for ext, args in \
six.iteritems(self.meta_metadata[main_key]['ext']):
if ext in exclude:
continue
FieldParser.field_extensions()[ext]\
.evaluate(self, main_key, action, args)
if 'decorators' not in exclude:
for ext, args in \
six.iteritems(self.meta_metadata[main_key]['after']):
if ext in exclude:
continue
FieldParser.decorator_after_extensions()[ext]\
.evaluate(self, main_key, action, args)
return self._dict_bson[key]
else:
try:
rest_of_key = SmartDict.main_key_pattern.findall(key)[0]
except IndexError:
rest_of_key = ''
return self[
self._dict['__meta_metadata__']['__aliases__'][main_key] +
rest_of_key]
def __setitem__(self, key, value, extend=False, **kwargs):
"""Like in `dict.__setitem__`.
There are two possible scenarios, i) the key is already present in the
dictionary (also its metadata), therefore a simple set in the first
step, ii) or the key is not inside the dictionary, in this case the
metadata for the field is fetched before performing the rest of the
actions.
Like in `__getitem__` after setting the value to the BSON dictionary
the decorators and the extensions are evaluated and, if needed, the
JSON representation of `value` is set into the JSON dictionary.
:param key:
:param value:
:param extend: If `False` it behaves as `ditc.__setitem__`, if `True`
creates a list with the previous content and append `value` to it.
:param kwargs:Typically used to set:
* `action`, Whether we are performing a `set` or a `get`.
*`exclude`, from the list of extensions and decorators excludes the
ones that are not required.
"""
main_key = SmartDict.main_key_pattern.sub('', key)
# If we have meta_metadata for the main key go ahead
if main_key in self.meta_metadata:
self._dict_bson.__setitem__(key, value, extend)
# Othewise we need the meta_metadata
else:
Reader.set(self, main_key)
self._dict_bson.__setitem__(key, value, extend, **kwargs)
action = kwargs.get('action', 'set')
exclude = kwargs.get('exclude', [])
if 'decorators' not in exclude:
for ext, args in \
six.iteritems(self.meta_metadata[main_key]['after']):
if ext in exclude:
continue
FieldParser.decorator_after_extensions()[ext]\
.evaluate(self, main_key, action, args)
if 'extensions' not in exclude:
for ext, args in \
six.iteritems(self.meta_metadata[main_key]['ext']):
if ext in exclude:
continue
FieldParser.field_extensions()[ext]\
.evaluate(self, main_key, action, args)
def __str__(self):
"""Representation of the object **without** the meta_metadata."""
return self.dumps(without_meta_metadata=True).__str__()
def __repr__(self):
"""Full string representation of the JSON object."""
return self._dict.__repr__()
def __delitem__(self, key):
"""Delete on key from the dictionary and its meta_metadata.
Note: It only works with default python keys
"""
self._dict.__delitem__(key)
del self._dict['__meta_metadata__'][key]
try:
del self._dict_bson[key]
except KeyError:
pass
def get(self, key, default=None, reset=False, **kwargs):
"""Like in `dict.get`."""
try:
return self.__getitem__(key, reset, **kwargs)
except (KeyError, IndexError):
return default
def items(self, without_meta_metadata=False):
"""Like in `dict.items`."""
for key in self.keys():
if key == '__meta_metadata__' and without_meta_metadata:
continue
yield (key, self[key])
iteritems = items
def keys(self, without_meta_metadata=False):
"""Like in `dict.keys`."""
for key in super(SmartJson, self).keys():
if key == '__meta_metadata__' and without_meta_metadata:
continue
yield key
def get_blob(self, *args, **kwargs):
"""To be override in the specific class.
Should look for the original version of the file where the json came
from.
"""
raise NotImplementedError()
def dumps(self,
without_meta_metadata=False,
with_calculated_fields=False,
clean=False,
keywords=None,
filter_hidden=False):
"""Create the JSON friendly representation of the current object.
:param without_meta_metadata: by default ``False``, if set to ``True``
all the ``__meta_metadata__`` will be removed from the output.
:param wit_calculated_fields: by default the calculated fields are not
dump, if they are needed in the output set it to ``True``
:param clean: if set to ``True`` all the keys stating with ``_`` will
be removed from the ouput
:param keywords: list of keywords to dump. if None, return all
:return: JSON friendly object
"""
dict_ = copy.copy(self._dict)
filter_keywords = keywords is not None and any(keywords)
if without_meta_metadata:
del dict_['__meta_metadata__']
# skip the dict iteration
if not any(
[clean, filter_keywords, filter_hidden, with_calculated_fields]):
return dict_
for key, value in six.iteritems(self._dict):
if (clean and key.startswith('_')
) or (filter_keywords and key not in keywords) or (
filter_hidden
and self.meta_metadata.get(key, {}).get('hidden', False)):
del dict_[key]
continue
if with_calculated_fields:
if value is None and \
self.meta_metadata[key]['type'] == 'calculated':
dict_[key] = self[key]
return dict_
def loads(self,
without_meta_metadata=False,
with_calculated_fields=True,
clean=False):
"""Create the BSON representation of the current object.
:param without_meta_metadata: if set to ``True`` all the
``__meta_metadata__`` will be removed from the output.
:param wit_calculated_fields: by default the calculated fields are in
the output, if they are not needed set it to ``False``
:param clean: if set to ``True`` all the keys stating with ``_`` will
be removed from the ouput
:return: JSON friendly object
"""
dict_ = dict()
for key in self.keys():
dict_[key] = self[key]
if without_meta_metadata:
del dict_['__meta_metadata__']
if not with_calculated_fields:
for key in self.keys():
if self.meta_metadata[key]['type'] == 'calculated':
del dict_[key]
if clean:
for key in list(dict_.keys()):
if key.startswith('_'):
del dict_[key]
return dict_
def produce(self, producer_code, fields=None):
"""Create a different flavor of `JSON` depending on `procuder_code`.
:param producer_code: One of the possible producers listed in the
`producer` section inside the field definitions.
:param fields: List of fields that should be present in the output, if
`None` all fields from `self` will be used.
:return: It depends on each producer, see producer folder inside
`jsonext`, typically `dict`.
"""
return producers[producer_code](self, fields=fields)
def set_default_values(self, fields=None):
"""Set default value for the fields using the schema definition.
:param fields: List of fields to set the default value, if `None` all.
"""
raise NotImplementedError('Missing implementation in this version')
def validate(self, validator=None):
"""Validate using current JSON content using Cerberus.
See: (Cerberus)[http://cerberus.readthedocs.org/en/latest].
:param validator: Validator to be used, if `None`
:class:`~.validator.Validator`
"""
if validator is None:
from .validator import Validator as validator
schema = dict()
model_fields = ModelParser.resolve_models(
self.model_info.names,
self.additional_info.namespace).get('fields', {})
for field in self.keys():
if not field == '__meta_metadata__' and \
field not in model_fields and \
self.meta_metadata[field]['json_id'] not in model_fields:
model_fields[field] = self.meta_metadata[field]['json_id']
for json_id in model_fields.keys():
try:
schema.update(
FieldParser.field_definitions(
self.additional_info.namespace)[json_id].get(
'schema', {}))
except (TypeError, KeyError):
pass
_validator = validator(schema=schema)
_validator.validate(self)
return _validator.errors
def test_smart_dict(self):
d = SmartDict()
d['foo'] = {'a': 'world', 'b': 'hello'}
d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
self.assertEqual(d.keys(), ['a', 'foo'])
self.assertTrue('foo.a' in d)
del d['foo']
self.assertEqual(d.keys(), ['a'])
self.assertEqual(d['a'], [{'b': 1}, {'b': 2}, {'b': 3}])
self.assertEqual(d['a[0]'], {'b': 1})
self.assertEqual(d['a.b'], [1, 2, 3])
self.assertEqual(d['a[1:]'], [{'b': 2}, {'b': 3}])
d.set('a', {'b': 4}, extend=True)
self.assertEqual(d['a'], [{'b': 1}, {'b': 2}, {'b': 3}, {'b': 4}])
d.set('a', [{'b': 1}, {'b': 2}, {'b': 3}], extend=False)
self.assertEqual(d['a'], [{'b': 1}, {'b': 2}, {'b': 3}])
self.assertEqual(d.get('does not exists'), None)
d = SmartDict()
d.set('a.b.c[n]', 'foo', True)
self.assertEqual(d['a.b.c'], ['foo'])
d.set('a.b.c[n]', 'bar', True)
self.assertEqual(d['a.b.c'], ['foo', 'bar'])
d.set('a.b.c', ['foo'], False)
self.assertEqual(d['a.b.c'], ['foo'])
class SmartJson(SmartDict):
"""Base class for Json structures."""
def __init__(self, json=None, set_default_values=False,
process_model_info=False, **kwargs):
"""If no JSON, a new structure will be created.
Typically the JSON structure needs a few mandatory fields and subfields
that are created here, the final JSON should look like this::
{
'__meta_metadata': {
'__additional_info__': {...},
'__model_info__': {'model_names': [...], ....},
'__aliases__': {...},
'__errors__': [...],
'__continuable_errors__': []
}
}
The content of `__additional_info__` is, usually, the content of
kwargs.
This object maintains two different dictionaries, one with the pure
JSON representation, `_dict` and a second one with the BSON
representation of the former. This behavior helps whenever dealing with
JSON databases like PostgreSQL.
The content of the BSON dictionary is typically the same as in the JSON
one, only if the `json` section is described in the definition of the
field the content might change, a good example of this behavior are the
date fields.
See :class:`~.jsonext.parsers.json_extra_parser:JsonExtraParser`.
:param json: JSON to build the dictionary.
:param set_defaul_values: If `True` default values will be set.
:param process_model_info: If `True` all model info will be parsed.
:param kwargs: Everything that would be useful inside
`__additional_info__`. Although it could be used also for:
* `model=['model1', 'model2']`, in this case this key will be
deleted from `kwargs` and used as model names inside
`__model_info__`.
"""
super(SmartJson, self).__init__(json)
self._dict_bson = SmartDict()
if not json or '__meta_metadata__' not in json:
model_names = kwargs.get('model', ['__default__', ])
if 'model' in kwargs:
del kwargs['model']
if isinstance(model_names, six.string_types):
model_names = [model_names, ]
self._dict['__meta_metadata__'] = dict()
self._dict['__meta_metadata__']['__additional_info__'] = kwargs
self._dict['__meta_metadata__']['__model_info__'] = \
dict(names=model_names)
self._dict['__meta_metadata__']['__aliases__'] = dict()
self._dict['__meta_metadata__']['__errors__'] = list()
self._dict['__meta_metadata__']['__continuable_errors__'] = list()
if process_model_info:
Reader.process_model_info(self)
if set_default_values:
self.set_default_values()
@property
def additional_info(self):
"""Shortcut to `__meta_metadata__.__additional_info__`."""
return DotableDict(self['__meta_metadata__']['__additional_info__'])
@property
def errors(self):
"""Shortcut to `__meta_metadata__.__errors__`."""
return self._dict['__meta_metadata__']['__errors__']
@property
def continuable_errors(self):
"""Shortcut to `__meta_metadata__.__continuable_errors__`."""
return self._dict['__meta_metadata__']['__continuable_errors__']
@property
def meta_metadata(self):
"""Shortcut to `__meta_metadata__`."""
return DotableDict(self._dict['__meta_metadata__'])
@property
def model_info(self):
"""Shortcut to `__meta_metadata__.__model_info__`."""
return DotableDict(self._dict['__meta_metadata__']['__model_info__'])
def __getitem__(self, key, reset=False, **kwargs):
"""Like in `dict.__getitem__`.
First it tries to load the value from the value from the BSON object,
if it fails, or reset is set to `True`, looks into the JSON object for
the `key`, applies all the extensions and decorators and return the
value that is stored in the BSON object as a cached version.
If the key is not found inside the dictionary, it tries before raising
`KeyError` to figure out if it is dealing with an alias.
:param key:
:param reset: If the key corresponds to a field calculated on the fly
the value will be calculated again.
:param kwargs: Typically used to set:
* `action`, Whether we are performing a `set` or a `get`.
*`exclude`, from the list of extensions and decorators excludes the
ones that are not required.
:return: Like in `dict.__getitem__`
"""
try:
value = self._dict_bson[key]
if value and not reset:
return value
except KeyError:
# Try to find the key inside the json dict and load it
pass
main_key = SmartDict.main_key_pattern.sub('', key)
if main_key == '__meta_metadata__':
return super(SmartJson, self).__getitem__(key)
elif main_key in self._dict:
if main_key not in self.meta_metadata:
Reader.set(self, main_key)
action = kwargs.get('action', 'get')
exclude = kwargs.get('exclude', [])
if 'extensions' not in exclude:
for ext, args in \
six.iteritems(self.meta_metadata[main_key]['ext']):
if ext in exclude:
continue
FieldParser.field_extensions()[ext]\
.evaluate(self, main_key, action, args)
if 'decorators' not in exclude:
for ext, args in \
six.iteritems(self.meta_metadata[main_key]['after']):
if ext in exclude:
continue
FieldParser.decorator_after_extensions()[ext]\
.evaluate(self, main_key, action, args)
return self._dict_bson[key]
else:
try:
rest_of_key = SmartDict.main_key_pattern.findall(key)[0]
except IndexError:
rest_of_key = ''
return self[
self._dict['__meta_metadata__']['__aliases__'][main_key]
+ rest_of_key]
def __setitem__(self, key, value, extend=False, **kwargs):
"""Like in `dict.__setitem__`.
There are two possible scenarios, i) the key is already present in the
dictionary (also its metadata), therefore a simple set in the first
step, ii) or the key is not inside the dictionary, in this case the
metadata for the field is fetched before performing the rest of the
actions.
Like in `__getitem__` after setting the value to the BSON dictionary
the decorators and the extensions are evaluated and, if needed, the
JSON representation of `value` is set into the JSON dictionary.
:param key:
:param value:
:param extend: If `False` it behaves as `ditc.__setitem__`, if `True`
creates a list with the previous content and append `value` to it.
:param kwargs:Typically used to set:
* `action`, Whether we are performing a `set` or a `get`.
*`exclude`, from the list of extensions and decorators excludes the
ones that are not required.
"""
main_key = SmartDict.main_key_pattern.sub('', key)
# If we have meta_metadata for the main key go ahead
if main_key in self.meta_metadata:
self._dict_bson.__setitem__(key, value, extend)
# Othewise we need the meta_metadata
else:
Reader.set(self, main_key)
self._dict_bson.__setitem__(key, value, extend, **kwargs)
action = kwargs.get('action', 'set')
exclude = kwargs.get('exclude', [])
if 'decorators' not in exclude:
for ext, args in \
six.iteritems(self.meta_metadata[main_key]['after']):
if ext in exclude:
continue
FieldParser.decorator_after_extensions()[ext]\
.evaluate(self, main_key, action, args)
if 'extensions' not in exclude:
for ext, args in \
six.iteritems(self.meta_metadata[main_key]['ext']):
if ext in exclude:
continue
FieldParser.field_extensions()[ext]\
.evaluate(self, main_key, action, args)
def __str__(self):
"""Representation of the object **without** the meta_metadata."""
return self.dumps(without_meta_metadata=True).__str__()
def __repr__(self):
"""Full string representation of the JSON object."""
return self._dict.__repr__()
def __delitem__(self, key):
"""Delete on key from the dictionary and its meta_metadata.
Note: It only works with default python keys
"""
self._dict.__delitem__(key)
del self._dict['__meta_metadata__'][key]
try:
del self._dict_bson[key]
except KeyError:
pass
def get(self, key, default=None, reset=False, **kwargs):
"""Like in `dict.get`."""
try:
return self.__getitem__(key, reset, **kwargs)
except (KeyError, IndexError):
return default
def items(self, without_meta_metadata=False):
"""Like in `dict.items`."""
for key in self.keys():
if key == '__meta_metadata__' and without_meta_metadata:
continue
yield (key, self[key])
iteritems = items
def keys(self, without_meta_metadata=False):
"""Like in `dict.keys`."""
for key in super(SmartJson, self).keys():
if key == '__meta_metadata__' and without_meta_metadata:
continue
yield key
def get_blob(self, *args, **kwargs):
"""To be override in the specific class.
Should look for the original version of the file where the json came
from.
"""
raise NotImplementedError()
def dumps(self, without_meta_metadata=False, with_calculated_fields=False,
clean=False, keywords=None, filter_hidden=False):
"""Create the JSON friendly representation of the current object.
:param without_meta_metadata: by default ``False``, if set to ``True``
all the ``__meta_metadata__`` will be removed from the output.
:param wit_calculated_fields: by default the calculated fields are not
dump, if they are needed in the output set it to ``True``
:param clean: if set to ``True`` all the keys stating with ``_`` will
be removed from the ouput
:param keywords: list of keywords to dump. if None, return all
:return: JSON friendly object
"""
dict_ = copy.copy(self._dict)
filter_keywords = keywords is not None and any(keywords)
if without_meta_metadata:
del dict_['__meta_metadata__']
# skip the dict iteration
if not any([clean, filter_keywords, filter_hidden,
with_calculated_fields]):
return dict_
for key, value in six.iteritems(self._dict):
if (clean and key.startswith('_')) or (
filter_keywords and key not in keywords) or (
filter_hidden and self.meta_metadata.get(key, {}).get(
'hidden', False)):
del dict_[key]
continue
if with_calculated_fields:
if value is None and \
self.meta_metadata[key]['type'] == 'calculated':
dict_[key] = self[key]
return dict_
def loads(self, without_meta_metadata=False, with_calculated_fields=True,
clean=False):
"""Create the BSON representation of the current object.
:param without_meta_metadata: if set to ``True`` all the
``__meta_metadata__`` will be removed from the output.
:param wit_calculated_fields: by default the calculated fields are in
the output, if they are not needed set it to ``False``
:param clean: if set to ``True`` all the keys stating with ``_`` will
be removed from the ouput
:return: JSON friendly object
"""
dict_ = dict()
for key in self.keys():
dict_[key] = self[key]
if without_meta_metadata:
del dict_['__meta_metadata__']
if not with_calculated_fields:
for key in self.keys():
if self.meta_metadata[key]['type'] == 'calculated':
del dict_[key]
if clean:
for key in list(dict_.keys()):
if key.startswith('_'):
del dict_[key]
return dict_
def produce(self, producer_code, fields=None):
"""Create a different flavor of `JSON` depending on `procuder_code`.
:param producer_code: One of the possible producers listed in the
`producer` section inside the field definitions.
:param fields: List of fields that should be present in the output, if
`None` all fields from `self` will be used.
:return: It depends on each producer, see producer folder inside
`jsonext`, typically `dict`.
"""
return producers[producer_code](self, fields=fields)
def set_default_values(self, fields=None):
"""Set default value for the fields using the schema definition.
:param fields: List of fields to set the default value, if `None` all.
"""
raise NotImplementedError('Missing implementation in this version')
def validate(self, validator=None):
"""Validate using current JSON content using Cerberus.
See: (Cerberus)[http://cerberus.readthedocs.org/en/latest].
:param validator: Validator to be used, if `None`
:class:`~.validator.Validator`
"""
if validator is None:
from .validator import Validator as validator
schema = dict()
model_fields = ModelParser.resolve_models(
self.model_info.names,
self.additional_info.namespace).get('fields', {})
for field in self.keys():
if not field == '__meta_metadata__' and \
field not in model_fields and \
self.meta_metadata[field]['json_id'] not in model_fields:
model_fields[field] = self.meta_metadata[field]['json_id']
for json_id in model_fields.keys():
try:
schema.update(FieldParser.field_definitions(
self.additional_info.namespace)[json_id].get('schema', {}))
except (TypeError, KeyError):
pass
_validator = validator(schema=schema)
_validator.validate(self)
return _validator.errors
def __init__(self, json=None, set_default_values=False,
process_model_info=False, **kwargs):
"""If no JSON, a new structure will be created.
Typically the JSON structure needs a few mandatory fields and subfields
that are created here, the final JSON should look like this::
{
'__meta_metadata': {
'__additional_info__': {...},
'__model_info__': {'model_names': [...], ....},
'__aliases__': {...},
'__errors__': [...],
'__continuable_errors__': []
}
}
The content of `__additional_info__` is, usually, the content of
kwargs.
This object maintains two different dictionaries, one with the pure
JSON representation, `_dict` and a second one with the BSON
representation of the former. This behavior helps whenever dealing with
JSON databases like PostgreSQL.
The content of the BSON dictionary is typically the same as in the JSON
one, only if the `json` section is described in the definition of the
field the content might change, a good example of this behavior are the
date fields.
See :class:`~.jsonext.parsers.json_extra_parser:JsonExtraParser`.
:param json: JSON to build the dictionary.
:param set_defaul_values: If `True` default values will be set.
:param process_model_info: If `True` all model info will be parsed.
:param kwargs: Everything that would be useful inside
`__additional_info__`. Although it could be used also for:
* `model=['model1', 'model2']`, in this case this key will be
deleted from `kwargs` and used as model names inside
`__model_info__`.
"""
super(SmartJson, self).__init__(json)
self._dict_bson = SmartDict()
if not json or '__meta_metadata__' not in json:
model_names = kwargs.get('model', ['__default__', ])
if 'model' in kwargs:
del kwargs['model']
if isinstance(model_names, six.string_types):
model_names = [model_names, ]
self._dict['__meta_metadata__'] = dict()
self._dict['__meta_metadata__']['__additional_info__'] = kwargs
self._dict['__meta_metadata__']['__model_info__'] = \
dict(names=model_names)
self._dict['__meta_metadata__']['__aliases__'] = dict()
self._dict['__meta_metadata__']['__errors__'] = list()
self._dict['__meta_metadata__']['__continuable_errors__'] = list()
if process_model_info:
Reader.process_model_info(self)
if set_default_values:
self.set_default_values()
def test_smart_insert_special_chars(self):
d = SmartDict({'a': 'world', 'b': 'hello'})
self.assertRaises(KeyError, setitem, d, ".", "dot")
self.assertRaises(KeyError, setitem, d, "[", "open bracket")
self.assertRaises(KeyError, setitem, d, "]", "close bracket")
