def test_field_name_model_based(self):
"""JSONAlchemy - field name model based"""
field_model_def = self.model_parser.field_definition_model_based(
'title', 'test_model')
field_def = self.field_parser.field_definitions()['title_title']
value = {'a': 'Awesome title', 'b': 'sub title', 'k': 'form'}
from jsonalchemy.utils import try_to_eval
self.assertEqual(
try_to_eval(field_model_def['rules'][
'marc'][0]['function'], value=value),
try_to_eval(field_def['rules']['marc'][0]['function'],
value=value))
def test_field_name_model_based(self):
"""JSONAlchemy - field name model based"""
field_model_def = self.model_parser.field_definition_model_based(
'title', 'test_model')
field_def = self.field_parser.field_definitions()['title_title']
value = {'a': 'Awesome title', 'b': 'sub title', 'k': 'form'}
from jsonalchemy.utils import try_to_eval
self.assertEqual(
try_to_eval(field_model_def['rules']['marc'][0]['function'],
value=value),
try_to_eval(field_def['rules']['marc'][0]['function'],
value=value))
def _apply_virtual_rules(self, json_id, field_name, rule):
"""Try to apply either a 'derived' or 'calculated' rule.
:param json_id: Name os the json field in the configuration file.
:param field_name: Final name of the field, taken from the model
definiti, if any, otherwise is equal to the `json_id`
:param rule: Current rule for the `json_id`
"""
field_defs = []
field_defs.append(('calculated', rule['rules'].get('calculated', [])))
field_defs.append(('derived', rule['rules'].get('derived', [])))
for (field_type, _field_def) in field_defs:
for field_def in _field_def:
if not self._evaluate_before_decorators(field_def):
continue
try:
value = try_to_eval(
field_def['function'],
self.metadata.functions,
self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(json_id, field_name,
field_type, field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(
field_name, value, extend=False,
exclude=['decorators', 'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for virtual "
"field '%s'. \n%s" % (field_name, str(e)),)
def _apply_virtual_rules(self, json_id, field_name, rule):
"""Try to apply either a 'derived' or 'calculated' rule.
:param json_id: Name os the json field in the configuration file.
:param field_name: Final name of the field, taken from the model
definiti, if any, otherwise is equal to the `json_id`
:param rule: Current rule for the `json_id`
"""
field_defs = []
field_defs.append(('calculated', rule['rules'].get('calculated', [])))
field_defs.append(('derived', rule['rules'].get('derived', [])))
for (field_type, _field_def) in field_defs:
for field_def in _field_def:
if not self._evaluate_before_decorators(field_def):
continue
try:
value = try_to_eval(field_def['function'],
self.metadata.functions,
self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(json_id, field_name,
field_type, field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(
field_name,
value,
extend=False,
exclude=['decorators', 'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for virtual "
"field '%s'. \n%s" % (field_name, str(e)), )
def evaluate(cls, json, field_name, action, args):
"""
Applies the connect funtion with json, field_name and action parameters
if any functions availabe, otherwise it will put the content of the
current field into the connected one.
"""
if action == 'get':
return
for info in args:
if info['update_function'] is None:
json.__setitem__(info['connected_field'],
json[field_name],
exclude='connect')
else:
try_to_eval(info['update_function'], json.metadata.functions)(
json, field_name, info['connected_field'], action)
def evaluate(cls, value, metadata, args):
"""Evaluate ``args`` with the master value from the input.
:returns: a boolean depending on evaluated ``value``.
"""
evaluated = try_to_eval(args, metadata.functions, value=value)
if not isinstance(evaluated, (list, tuple)):
return evaluated
else:
return all(evaluated)
def evaluate(cls, json, field_name, action, args):
"""
Applies the connect funtion with json, field_name and action parameters
if any functions availabe, otherwise it will put the content of the
current field into the connected one.
"""
if action == 'get':
return
for info in args:
if info['update_function'] is None:
json.__setitem__(info['connected_field'],
json[field_name],
exclude='connect')
else:
try_to_eval(info['update_function'],
json.metadata.functions)(json, field_name,
info['connected_field'],
action)
def _apply_rules(self, json_id, field_name, rule):
"""Override default behavior.
See :meth:`~jsonalchemy.readers.Reader._apply_rules`.
Tries to apply all the rules defined for marc and as soon as it can
apply one it stops the process.
It also keeps the marc_tag that comes from
:meth:`._get_elements_from_blob`
:param json_id: as in
:meth:`~jsonalchemy.readers.Reader._apply_rules`
:param field_name: as in
:meth:`~jsonalchemy.readers.Reader._apply_rules`
:param rule: as in
:meth:`~jsonalchemy.readers.Reader._apply_rules`
"""
for field_def in rule['rules'].get(
self._json.additional_info.master_format, []):
if not self._evaluate_before_decorators(field_def):
continue
marc_tag, elements = self._get_elements_from_blob(
field_def['source_tags'])
if not isinstance(elements, (list, tuple)):
elements = (elements, )
for element in elements:
tmp_field_def = copy.deepcopy(field_def)
tmp_field_def['source_tags'] = [
marc_tag,
]
if not self._evaluate_on_decorators(tmp_field_def, element):
continue
try:
value = try_to_eval(tmp_field_def['function'],
self.metadata.functions,
value=element,
self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(json_id, field_name,
'creator', tmp_field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(
field_name,
value,
extend=True,
exclude=['decorators', 'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for field "
"'%s' with value '%s'. \n%s" %
(field_name, element, str(e)), )
if field_name in self._json._dict_bson:
break
def evaluate(cls, reader, args):
"""Evaluate parser.
This is a special case where the real evaluation of the decorator
is happening before the evaluation.
"""
evaluated = try_to_eval(args,
reader.metadata.functions,
self=reader._json)
if not isinstance(evaluated, (list, tuple)):
return evaluated
else:
return all(evaluated)
def evaluate(cls, reader, args):
"""Evaluate parser.
This is a special case where the real evaluation of the decorator
is happening before the evaluation.
"""
evaluated = try_to_eval(
args, reader.metadata.functions,
self=reader._json)
if not isinstance(evaluated, (list, tuple)):
return evaluated
else:
return all(evaluated)
def _apply_rules(self, json_id, field_name, rule):
"""Override default behavior.
See :meth:`~jsonalchemy.readers.Reader._apply_rules`.
Tries to apply all the rules defined for marc and as soon as it can
apply one it stops the process.
It also keeps the marc_tag that comes from
:meth:`._get_elements_from_blob`
:param json_id: as in
:meth:`~jsonalchemy.readers.Reader._apply_rules`
:param field_name: as in
:meth:`~jsonalchemy.readers.Reader._apply_rules`
:param rule: as in
:meth:`~jsonalchemy.readers.Reader._apply_rules`
"""
for field_def in rule['rules'].get(
self._json.additional_info.master_format, []):
if not self._evaluate_before_decorators(field_def):
continue
marc_tag, elements = self._get_elements_from_blob(
field_def['source_tags'])
if not isinstance(elements, (list, tuple)):
elements = (elements, )
for element in elements:
tmp_field_def = copy.deepcopy(field_def)
tmp_field_def['source_tags'] = [marc_tag, ]
if not self._evaluate_on_decorators(tmp_field_def, element):
continue
try:
value = try_to_eval(
tmp_field_def['function'],
self.metadata.functions,
value=element, self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(json_id, field_name,
'creator', tmp_field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(field_name, value, extend=True,
exclude=['decorators',
'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for field "
"'%s' with value '%s'. \n%s"
% (field_name, element, str(e)),)
if field_name in self._json._dict_bson:
break
def evaluate(cls, obj, args):
"""Extend the incoming object with all the new things from args."""
if args is None:
return
extensions = []
for ext in args:
try:
if ':' in ext:
package, attr = ext.split(':')
else:
package, attr = ext.rsplit('.', 1)
extensions.append(
getattr(importlib.import_module(package), attr))
except (ImportError, AttributeError, KeyError):
extensions.append(try_to_eval(ext))
extensions.append(obj.__class__)
obj.__class__ = type(obj.__class__.__name__, tuple(extensions), {})
def evaluate(cls, obj, args):
"""Extend the incoming object with all the new things from args."""
if args is None:
return
extensions = []
for ext in args:
try:
if ':' in ext:
package, attr = ext.split(':')
else:
package, attr = ext.rsplit('.', 1)
extensions.append(
getattr(importlib.import_module(package), attr)
)
except (ImportError, AttributeError, KeyError):
extensions.append(try_to_eval(ext))
extensions.append(obj.__class__)
obj.__class__ = type(obj.__class__.__name__, tuple(extensions), {})
def _apply_rules(self, json_id, field_name, rule):
"""Try to apply a 'creator' rule.
:param json_id: Name os the json field in the configuration file.
:param field_name: Final name of the field, taken from the model
definiti, if any, otherwise is equal to the `json_id`
:param rule: Current rule for the `json_id`
"""
for field_def in rule['rules'].get(
self._json.additional_info.master_format, []):
if not self._evaluate_before_decorators(field_def):
continue
for elements in \
self._get_elements_from_blob(field_def['source_tags']):
if not isinstance(elements, (list, tuple)):
elements = (elements, )
for element in elements:
if not self._evaluate_on_decorators(field_def, element):
continue
try:
value = try_to_eval(field_def['function'],
self.metadata.functions,
value=element,
self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(
json_id, field_name, 'creator', field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(
field_name,
value,
extend=True,
exclude=['decorators', 'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for field "
"'%s' with value '%s'. \n%s" %
(field_name, element, str(e)), )
def evaluate(cls, json, field_name, action, args):
"""Evaluate the parser.
When getting a json field compare the timestamp and the lifetime of it
and, if it the lifetime is over calculate its value again.
If the value of the field has changed since the last time it gets
updated in the DB.
"""
# if cls.__cache is None:
# cls.__cache = import_string(cfg.get('CFG_JSONALCHEMY_CACHE',
# 'invenio.ext.cache:cache'))
#
# @cls.__cache.memoize(timeout=args)
def memoize(_id, field_name):
func = reduce(lambda obj, key: obj[key],
json.meta_metadata[field_name]['function'],
json.metadata.field_parser.field_definitions())
return try_to_eval(func, json.metadata.functions, self=json)
if args == cls.DEFAULT_TIMEOUT:
return
if action == 'get':
if args == 0: # No cached version is stored, retrieve it
func = reduce(lambda obj, key: obj[key],
json.meta_metadata[field_name]['function'],
json.metadata.field_parser.field_definitions())
json._dict_bson[field_name] = try_to_eval(
func,
json.metadata.functions,
self=json)
else:
json._dict_bson[field_name] = memoize(json.get('_id'),
field_name)
elif action == 'set':
if args >= 0: # Don't store anything
json._dict_bson[field_name] = None
def _apply_rules(self, json_id, field_name, rule):
"""Try to apply a 'creator' rule.
:param json_id: Name os the json field in the configuration file.
:param field_name: Final name of the field, taken from the model
definiti, if any, otherwise is equal to the `json_id`
:param rule: Current rule for the `json_id`
"""
for field_def in rule['rules'].get(
self._json.additional_info.master_format, []):
if not self._evaluate_before_decorators(field_def):
continue
for elements in \
self._get_elements_from_blob(field_def['source_tags']):
if not isinstance(elements, (list, tuple)):
elements = (elements, )
for element in elements:
if not self._evaluate_on_decorators(field_def, element):
continue
try:
value = try_to_eval(
field_def['function'],
self.metadata.functions,
value=element, self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(json_id, field_name,
'creator', field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(field_name, value, extend=True,
exclude=['decorators',
'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for field "
"'%s' with value '%s'. \n%s"
% (field_name, element, str(e)),)
def evaluate(cls, json, field_name, action, args):
"""Evaluate the parser.
When getting a json field compare the timestamp and the lifetime of it
and, if it the lifetime is over calculate its value again.
If the value of the field has changed since the last time it gets
updated in the DB.
"""
# if cls.__cache is None:
# cls.__cache = import_string(cfg.get('CFG_JSONALCHEMY_CACHE',
# 'invenio.ext.cache:cache'))
#
# @cls.__cache.memoize(timeout=args)
def memoize(_id, field_name):
func = reduce(lambda obj, key: obj[key],
json.meta_metadata[field_name]['function'],
json.metadata.field_parser.field_definitions())
return try_to_eval(func, json.metadata.functions, self=json)
if args == cls.DEFAULT_TIMEOUT:
return
if action == 'get':
if args == 0: # No cached version is stored, retrieve it
func = reduce(lambda obj, key: obj[key],
json.meta_metadata[field_name]['function'],
json.metadata.field_parser.field_definitions())
json._dict_bson[field_name] = try_to_eval(
func, json.metadata.functions, self=json)
else:
json._dict_bson[field_name] = memoize(json.get('_id'),
field_name)
elif action == 'set':
if args >= 0: # Don't store anything
json._dict_bson[field_name] = None
def produce(self, fields=None):
"""Export the json in marc format.
Produces a list of dictionaries will all the possible marc tags as keys.
:param fields: list of fields to include in the output, if None or
empty list all available tags will be included.
"""
if not fields:
fields = self.keys()
out = []
for field in fields:
if field.startswith('__') or self.get(field) is None:
continue
json_id = self.meta_metadata[field]['json_id']
values = self.get(field)
if not isinstance(values, (list, tuple)):
values = (values, )
for value in values:
try:
for rule in get_producer_rules(
json_id, 'json_for_marc',
self.metadata):
marc_tags = rule[0] if isinstance(rule[0], tuple) \
else (rule[0], )
if marc_tags and not any(
[re.match(m, t)
for m in marc_tags
for t in self.meta_metadata[field]['function']]):
# Not match, continue to next rule
continue
tmp_dict = dict()
for marc_tag, subfield in iteritems(rule[1]):
if len(marc_tag) == 1:
marc_tag = \
self.meta_metadata[field]['function'][0] + \
marc_tag
if not subfield:
tmp_dict[marc_tag] = value
else:
try:
tmp_dict[marc_tag] = value[subfield]
except:
try:
# Evaluate only non keyword values.
if subfield in __builtins__:
raise ImportError
tmp_dict[marc_tag] = try_to_eval(
subfield,
self.metadata.functions,
value=value,
self=self)
except ImportError:
pass
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce "
"'%s'.\n %s" % (field, str(e)))
if tmp_dict:
out.append(tmp_dict)
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce '%s'.\n %s"
% (field, str(e)))
return out
def memoize(_id, field_name):
func = reduce(lambda obj, key: obj[key],
json.meta_metadata[field_name]['function'],
json.metadata.field_parser.field_definitions())
return try_to_eval(func, json.metadata.functions, self=json)
def create_element(cls, rule, metadata):
"""Create the dictionary with the dump and load functions."""
return {'loads': try_to_eval(rule.json_ext.loads,
metadata.functions),
'dumps': try_to_eval(rule.json_ext.dumps,
metadata.functions)}
def memoize(_id, field_name):
func = reduce(lambda obj, key: obj[key],
json.meta_metadata[field_name]['function'],
json.metadata.field_parser.field_definitions())
return try_to_eval(func, json.metadata.functions, self=json)
def create_element(cls, rule, metadata):
"""Just evaluate the content of the schema to a python dictionary."""
return try_to_eval(rule.schema, metadata.functions)
def create_element(cls, rule, metadata):
"""Just evaluate the content of the schema to a python dictionary."""
return try_to_eval(rule.schema, metadata.functions)
def produce(self, fields=None):
"""Export the json in marc format.
Produces a list of dictionaries will all the possible marc tags as keys.
:param fields: list of fields to include in the output, if None or
empty list all available tags will be included.
"""
if not fields:
fields = self.keys()
out = []
for field in fields:
if field.startswith('__') or self.get(field) is None:
continue
json_id = self.meta_metadata[field]['json_id']
values = self.get(field)
if not isinstance(values, (list, tuple)):
values = (values, )
for value in values:
try:
for rule in get_producer_rules(json_id, 'json_for_marc',
self.metadata):
marc_tags = rule[0] if isinstance(rule[0], tuple) \
else (rule[0], )
if marc_tags and not any([
re.match(m, t) for m in marc_tags
for t in self.meta_metadata[field]['function']
]):
# Not match, continue to next rule
continue
tmp_dict = dict()
for marc_tag, subfield in iteritems(rule[1]):
if len(marc_tag) == 1:
marc_tag = \
self.meta_metadata[field]['function'][0] + \
marc_tag
if not subfield:
tmp_dict[marc_tag] = value
else:
try:
tmp_dict[marc_tag] = value[subfield]
except:
try:
# Evaluate only non keyword values.
if subfield in __builtins__:
raise ImportError
tmp_dict[marc_tag] = try_to_eval(
subfield,
self.metadata.functions,
value=value,
self=self)
except ImportError:
pass
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce "
"'%s'.\n %s" % (field, str(e)))
if tmp_dict:
out.append(tmp_dict)
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce '%s'.\n %s" %
(field, str(e)))
return out
