def create_element(cls, rule, namespace):
"""Creates the dictionary with the dump and load functions"""
return {'loads': try_to_eval(rule.json_ext.loads,
functions(namespace)),
'dumps': try_to_eval(rule.json_ext.dumps,
functions(namespace))}
def create_element(cls, rule, namespace):
"""Create the dictionary with the info from the configuratin file."""
element = dict()
element['mapping'] = try_to_eval(
rule.mapping, functions(namespace)) if rule.mapping else {}
element['facets'] = try_to_eval(
rule.facets, functions(namespace)) if rule.facets else {}
element['highlights'] = try_to_eval(
rule.highlights, functions(namespace)) if rule.highlights else {}
element['local_tokenizer'] = []
if rule.local_tokenizer:
for i in range(0, len(rule.local_tokenizer), 2):
element['local_tokenizer'].append(
(rule.local_tokenizer[i], rule.local_tokenizer[i + 1]))
return element
def _apply_virtual_rules(self, json_id, field_name, rule):
"""Try to apply either a 'derived' or 'calculated' rule.
:param json_id: Name os the json field in the configuration file.
:param field_name: Final name of the field, taken from the model
definiti, if any, otherwise is equal to the `json_id`
:param rule: Current rule for the `json_id`
"""
field_defs = []
field_defs.append(('calculated', rule['rules'].get('calculated', [])))
field_defs.append(('derived', rule['rules'].get('derived', [])))
for (field_type, _field_def) in field_defs:
for field_def in _field_def:
if not self._evaluate_before_decorators(field_def):
continue
try:
value = try_to_eval(
field_def['function'],
functions(self._json.additional_info.namespace),
self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(json_id, field_name,
field_type, field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(
field_name, value, extend=False,
exclude=['decorators', 'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for virtual "
"field '%s'. \n%s" % (field_name, str(e)),)
def test_field_name_model_based(self):
"""JSONAlchemy - field name model based"""
clean_field_model_definitions()
field_model_def = Field_parser.field_definition_model_based(
'title', 'test_model', 'testsuite')
field_def = Field_parser.field_definitions('testsuite')['title_title']
value = {'a': 'Awesome title', 'b': 'sub title', 'k': 'form'}
from invenio.base.utils import try_to_eval
self.assertEqual(
try_to_eval(field_model_def['rules'][
'marc'][0]['function'], value=value),
try_to_eval(field_def['rules']['marc'][0]['function'],
value=value))
clean_field_model_definitions()
def test_field_name_model_based(self):
"""JSONAlchemy - field name model based"""
clean_field_model_definitions()
field_model_def = Field_parser.field_definition_model_based(
'title', 'test_model', 'testsuite')
field_def = Field_parser.field_definitions('testsuite')['title_title']
value = {'a': 'Awesome title', 'b': 'sub title', 'k': 'form'}
from invenio.base.utils import try_to_eval
self.assertEqual(
try_to_eval(field_model_def['rules'][
'marc'][0]['function'], value=value),
try_to_eval(field_def['rules']['marc'][0]['function'],
value=value))
clean_field_model_definitions()
def _apply_virtual_rules(self, json_id, field_name, rule):
"""Try to apply either a 'derived' or 'calculated' rule.
:param json_id: Name os the json field in the configuration file.
:param field_name: Final name of the field, taken from the model
definiti, if any, otherwise is equal to the `json_id`
:param rule: Current rule for the `json_id`
"""
field_defs = []
field_defs.append(('calculated', rule['rules'].get('calculated', [])))
field_defs.append(('derived', rule['rules'].get('derived', [])))
for (field_type, _field_def) in field_defs:
for field_def in _field_def:
if not self._evaluate_before_decorators(field_def):
continue
try:
value = try_to_eval(
field_def['function'],
functions(self._json.additional_info.namespace),
self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(json_id, field_name,
field_type, field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(
field_name,
value,
extend=False,
exclude=['decorators', 'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for virtual "
"field '%s'. \n%s" % (field_name, str(e)), )
def memoize(_id, field_name):
func = reduce(
lambda obj, key: obj[key],
json.meta_metadata[field_name]['function'],
FieldParser.field_definitions(json.additional_info.namespace))
return try_to_eval(func,
functions(json.additional_info.namespace),
self=json)
def evaluate(cls, json, field_name, action, args):
"""
Applies the connect funtion with json, field_name and action parameters
if any functions availabe, otherwise it will put the content of the
current field into the connected one.
"""
if action == 'get':
return
from invenio.modules.jsonalchemy.registry import functions
for info in args:
if info['update_function'] is None:
json.__setitem__(info['connected_field'],
json[field_name],
exclude='connect')
else:
try_to_eval(info['update_function'],
functions(json.additional_info.namespace))(
json, field_name, info['connected_field'],
action)
def evaluate(cls, json, field_name, action, args):
"""
Applies the connect funtion with json, field_name and action parameters
if any functions availabe, otherwise it will put the content of the
current field into the connected one.
"""
if action == 'get':
return
from invenio.modules.jsonalchemy.registry import functions
for info in args:
if info['update_function'] is None:
json.__setitem__(info['connected_field'],
json[field_name],
exclude='connect')
else:
try_to_eval(info['update_function'],
functions(json.additional_info.namespace))(
json, field_name, info['connected_field'],
action)
def _apply_rules(self, json_id, field_name, rule):
"""Override default behavior.
See :meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules`.
Tries to apply all the rules defined for marc and as soon as it can
apply one it stops the process.
It also keeps the marc_tag that comes from
:meth:`._get_elements_from_blob`
:param json_id: as in
:meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules`
:param field_name: as in
:meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules`
:param rule: as in
:meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules`
"""
for field_def in rule['rules'].get(
self._json.additional_info.master_format, []):
if not self._evaluate_before_decorators(field_def):
continue
marc_tag, elements = self._get_elements_from_blob(
field_def['source_tags'])
if not isinstance(elements, (list, tuple)):
elements = (elements, )
for element in elements:
tmp_field_def = copy.deepcopy(field_def)
tmp_field_def['source_tags'] = [
marc_tag,
]
if not self._evaluate_on_decorators(tmp_field_def, element):
continue
try:
value = try_to_eval(
tmp_field_def['function'],
functions(self._json.additional_info.namespace),
value=element,
self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(json_id, field_name,
'creator', tmp_field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(
field_name,
value,
extend=True,
exclude=['decorators', 'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for field "
"'%s' with value '%s'. \n%s" %
(field_name, element, str(e)), )
if field_name in self._json._dict_bson:
break
def evaluate(cls, value, namespace, args):
"""Evaluate ``args`` with the master value from the input.
:returns: a boolean depending on evaluated ``value``.
"""
from invenio.modules.jsonalchemy.registry import functions
evaluated = try_to_eval(args, functions(namespace), value=value)
if not isinstance(evaluated, (list, tuple)):
return evaluated
else:
return all(evaluated)
def evaluate(cls, obj, args):
"""Extend the incoming object with all the new things from args"""
if args is None:
return
extensions = []
for ext in args:
try:
extensions.append(import_string(ext))
except ImportError:
extensions.append(try_to_eval(ext))
extensions.append(obj.__class__)
obj.__class__ = type(obj.__class__.__name__, tuple(extensions), {})
def evaluate(cls, reader, args):
"""
This is a special case where the real evaluation of the decorator
happened before the evaluation, :see:create_element
"""
from invenio.modules.jsonalchemy.registry import functions
evaluated = try_to_eval(args,
functions(reader._json.additional_info.namespace),
self=reader._json)
if not isinstance(evaluated, (list, tuple)):
return evaluated
else:
return all(evaluated)
def _apply_rules(self, json_id, field_name, rule):
"""Override default behavior.
See :meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules`.
Tries to apply all the rules defined for marc and as soon as it can
apply one it stops the process.
It also keeps the marc_tag that comes from
:meth:`._get_elements_from_blob`
:param json_id: as in
:meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules`
:param field_name: as in
:meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules`
:param rule: as in
:meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules`
"""
for field_def in rule['rules'].get(
self._json.additional_info.master_format, []):
if not self._evaluate_before_decorators(field_def):
continue
marc_tag, elements = self._get_elements_from_blob(
field_def['source_tags'])
if not isinstance(elements, (list, tuple)):
elements = (elements, )
for element in elements:
tmp_field_def = copy.deepcopy(field_def)
tmp_field_def['source_tags'] = [marc_tag, ]
if not self._evaluate_on_decorators(tmp_field_def, element):
continue
try:
value = try_to_eval(
tmp_field_def['function'],
functions(self._json.additional_info.namespace),
value=element, self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(json_id, field_name,
'creator', tmp_field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(field_name, value, extend=True,
exclude=['decorators',
'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for field "
"'%s' with value '%s'. \n%s"
% (field_name, element, str(e)),)
if field_name in self._json._dict_bson:
break
def evaluate(cls, reader, args):
"""Evaluate parser.
This is a special case where the real evaluation of the decorator
is happening before the evaluation.
"""
from invenio.modules.jsonalchemy.registry import functions
evaluated = try_to_eval(args,
functions(
reader._json.additional_info.namespace),
self=reader._json)
if not isinstance(evaluated, (list, tuple)):
return evaluated
else:
return all(evaluated)
def evaluate(cls, json, field_name, action, args):
"""Evaluate the parser.
When getting a json field compare the timestamp and the lifetime of it
and, if it the lifetime is over calculate its value again.
If the value of the field has changed since the last time it gets
updated in the DB.
"""
if cls.__cache is None:
cls.__cache = import_string(
cfg.get('CFG_JSONALCHEMY_CACHE', 'invenio.ext.cache:cache'))
@cls.__cache.memoize(timeout=args)
def memoize(_id, field_name):
func = reduce(
lambda obj, key: obj[key],
json.meta_metadata[field_name]['function'],
FieldParser.field_definitions(json.additional_info.namespace))
return try_to_eval(func,
functions(json.additional_info.namespace),
self=json)
if args == cls.DEFAULT_TIMEOUT:
return
if action == 'get':
if args == 0: # No cached version is stored, retrieve it
func = reduce(
lambda obj, key: obj[key],
json.meta_metadata[field_name]['function'],
FieldParser.field_definitions(
json.additional_info.namespace))
json._dict_bson[field_name] = try_to_eval(
func, functions(json.additional_info.namespace), self=json)
else:
json._dict_bson[field_name] = memoize(json.get('_id'),
field_name)
elif action == 'set':
if args >= 0: # Don't store anything
json._dict_bson[field_name] = None
def _apply_rules(self, json_id, field_name, rule):
"""Try to apply a 'creator' rule.
:param json_id: Name os the json field in the configuration file.
:param field_name: Final name of the field, taken from the model
definiti, if any, otherwise is equal to the `json_id`
:param rule: Current rule for the `json_id`
"""
for field_def in rule['rules'].get(
self._json.additional_info.master_format, []):
if not self._evaluate_before_decorators(field_def):
continue
for elements in \
self._get_elements_from_blob(field_def['source_tags']):
if not isinstance(elements, (list, tuple)):
elements = (elements, )
for element in elements:
if not self._evaluate_on_decorators(field_def, element):
continue
try:
value = try_to_eval(
field_def['function'],
functions(self._json.additional_info.namespace),
value=element,
self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(
json_id, field_name, 'creator', field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(
field_name,
value,
extend=True,
exclude=['decorators', 'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for field "
"'%s' with value '%s'. \n%s" %
(field_name, element, str(e)), )
def _apply_rules(self, json_id, field_name, rule):
"""Try to apply a 'creator' rule.
:param json_id: Name os the json field in the configuration file.
:param field_name: Final name of the field, taken from the model
definiti, if any, otherwise is equal to the `json_id`
:param rule: Current rule for the `json_id`
"""
for field_def in rule['rules'].get(
self._json.additional_info.master_format, []):
if not self._evaluate_before_decorators(field_def):
continue
for elements in \
self._get_elements_from_blob(field_def['source_tags']):
if not isinstance(elements, (list, tuple)):
elements = (elements, )
for element in elements:
if not self._evaluate_on_decorators(field_def, element):
continue
try:
value = try_to_eval(
field_def['function'],
functions(self._json.additional_info.namespace),
value=element, self=self._json)
self._remove_none_values(value)
info = self._find_field_metadata(json_id, field_name,
'creator', field_def)
self._json['__meta_metadata__'][field_name] = info
self._json.__setitem__(field_name, value, extend=True,
exclude=['decorators',
'extensions'])
except Exception as e:
self._json.errors.append(
"Rule Error - Unable to apply rule for field "
"'%s' with value '%s'. \n%s"
% (field_name, element, str(e)),)
def create_element(cls, rule, namespace):
"""Just evaluate the content of the schema to a python dictionary."""
return try_to_eval(rule.schema, functions(namespace))
def create_element(cls, rule, namespace):
"""Create the dictionary with the dump and load functions."""
return {
'loads': try_to_eval(rule.json_ext.loads, functions(namespace)),
'dumps': try_to_eval(rule.json_ext.dumps, functions(namespace))
}
def produce(self, fields=None):
"""Export the json for indexing purposes.
:param fields: list of fields to include in the output, if None or
empty list all available tags will be included.
"""
from invenio.base.utils import try_to_eval
from invenio.modules.jsonalchemy.parser import get_producer_rules
from invenio.modules.jsonalchemy.registry import functions
out = self.dumps(without_meta_metadata=True, with_calculated_fields=True,
keywords=fields)
for field, values in iteritems(out):
if field.startswith('__'):
continue
json_id = self.meta_metadata[field]['json_id']
tmp_dict = dict()
if not isinstance(values, (list, tuple)):
values = (values, )
for value in values:
try:
for rule in get_producer_rules(
json_id, 'json_for_indexer',
self.additional_info['namespace']):
# FIXME add support of indexer names.
# indexer_names = rule[0] if isinstance(rule[0], tuple) \
# else (rule[0], )
# if indexer_names and not any(
# [m == cfg['INDEXER_ENGINE'])
# for m in indexer_names])
# # Not match, continue to next rule
# continue
for subfield, value_or_function in iteritems(rule[1]):
try:
# Evaluate only non keyword values.
if value_or_function in __builtins__:
raise ImportError
tmp_dict[subfield] = try_to_eval(
value_or_function,
functions(
self.additional_info.namespace
),
value=value,
self=self)
except ImportError:
pass
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce "
"'%s'.\n %s" % (field, str(e)))
if tmp_dict:
if value is None:
value = tmp_dict
elif isinstance(value, dict):
value.update(tmp_dict)
else:
raise RuntimeError("Invalid field structure.")
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce '%s'.\n %s"
% (field, str(e)))
return out
def create_element(cls, rule, namespace):
"""Just evaluate the content of the schema to a python dictionary."""
return try_to_eval(rule.schema, functions(namespace))
def produce(self, fields=None):
"""Export the json in marc format.
Produces a list of dictionaries will all the possible marc tags as keys.
:param fields: list of fields to include in the output, if None or
empty list all available tags will be included.
"""
from invenio.base.utils import try_to_eval
from invenio.modules.jsonalchemy.parser import get_producer_rules
from invenio.modules.jsonalchemy.registry import functions
if not fields:
fields = self.keys()
out = []
for field in fields:
if field.startswith('__') or self.get(field) is None:
continue
json_id = self.meta_metadata[field]['json_id']
values = self.get(field)
if not isinstance(values, (list, tuple)):
values = (values, )
for value in values:
try:
for rule in get_producer_rules(
json_id, 'json_for_marc',
self.additional_info['namespace']):
marc_tags = rule[0] if isinstance(rule[0], tuple) \
else (rule[0], )
if marc_tags and not any([
re.match(m, t) for m in marc_tags
for t in self.meta_metadata[field]['function']
]):
# Not match, continue to next rule
continue
tmp_dict = dict()
for marc_tag, subfield in iteritems(rule[1]):
if len(marc_tag) == 1:
marc_tag = \
self.meta_metadata[field]['function'][0] + \
marc_tag
if not subfield:
tmp_dict[marc_tag] = value
else:
try:
tmp_dict[marc_tag] = value[subfield]
except:
try:
# Evaluate only non keyword values.
if subfield in __builtins__:
raise ImportError
tmp_dict[marc_tag] = try_to_eval(
subfield,
functions(
self.additional_info.namespace),
value=value,
self=self)
except ImportError:
pass
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce "
"'%s'.\n %s" % (field, str(e)))
if tmp_dict:
out.append(tmp_dict)
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce '%s'.\n %s" %
(field, str(e)))
return out
def produce(self, fields=None):
"""Export the record in JSON-LD format.
@param tags: list of tags to include in the output, if None or
empty list all available tags will be included.
"""
from invenio.base.utils import try_to_eval
from invenio.modules.jsonalchemy.parser import get_producer_rules
from invenio.modules.jsonalchemy.registry import functions
if not fields:
fields = self.keys()
out = {}
for field in fields:
if field.startswith('__') or self.get(field) is None:
continue
json_id = self.meta_metadata[field]['json_id']
values = self.get(field)
if not isinstance(values, (list, tuple)):
values = (values, )
for value in values:
try:
rules = get_producer_rules(json_id, 'json_for_ld', 'recordext')
for rule in rules:
tags = rule[0] if isinstance(rule[0], tuple) \
else (rule[0], )
if tags and not any([
tag in tags
for tag in self.meta_metadata[field]['function']]):
continue
tmp_dict = {}
for key, subfield in rule[1].items():
if not subfield:
tmp_dict[key] = value
else:
try:
tmp_dict[key] = value[subfield]
except:
try:
tmp_dict[key] = try_to_eval(
subfield,
functions(
self.additional_info.namespace
),
value=value, self=self)
except ImportError:
pass
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce "
"'%s'.\n %s" % (field, str(e)))
if tmp_dict:
for k, v in tmp_dict.items():
if isinstance(v, list):
if k not in out:
out[k] = []
for element in v:
out[k].append(element)
else:
out[k] = v
except KeyError as e:
self.continuable_errors.append(
"Producer CError - Unable to produce '%s'"
" (No rule found).\n %s"
% (field, str(e)))
return out
def produce(self, fields=None):
"""Export the json in marc format.
Produces a list of dictionaries will all the possible marc tags as keys.
:param fields: list of fields to include in the output, if None or
empty list all available tags will be included.
"""
from invenio.base.utils import try_to_eval
from invenio.modules.jsonalchemy.parser import get_producer_rules
from invenio.modules.jsonalchemy.registry import functions
if not fields:
fields = self.keys()
out = []
for field in fields:
if field.startswith('__') or self.get(field) is None:
continue
json_id = self.meta_metadata[field]['json_id']
values = self.get(field)
if not isinstance(values, (list, tuple)):
values = (values, )
for value in values:
try:
for rule in get_producer_rules(
json_id, 'json_for_marc',
self.additional_info['namespace']):
marc_tags = rule[0] if isinstance(rule[0], tuple) \
else (rule[0], )
if marc_tags and not any(
[re.match(m, t)
for m in marc_tags
for t in self.meta_metadata[field]['function']]):
# Not match, continue to next rule
continue
tmp_dict = dict()
for marc_tag, subfield in iteritems(rule[1]):
if len(marc_tag) == 1:
marc_tag = \
self.meta_metadata[field]['function'][0] + \
marc_tag
if not subfield:
tmp_dict[marc_tag] = value
else:
try:
tmp_dict[marc_tag] = value[subfield]
except:
try:
# Evaluate only non keyword values.
if subfield in __builtins__:
raise ImportError
tmp_dict[marc_tag] = try_to_eval(
subfield,
functions(
self.additional_info.namespace
),
value=value,
self=self)
except ImportError:
pass
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce "
"'%s'.\n %s" % (field, str(e)))
if tmp_dict:
out.append(tmp_dict)
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce '%s'.\n %s"
% (field, str(e)))
return out
def produce(self, fields=None):
"""Export the record in JSON-LD format.
@param tags: list of tags to include in the output, if None or
empty list all available tags will be included.
"""
from invenio.base.utils import try_to_eval
from invenio.modules.jsonalchemy.parser import get_producer_rules
from invenio.modules.jsonalchemy.registry import functions
if not fields:
fields = self.keys()
out = {}
for field in fields:
if field.startswith('__') or self.get(field) is None:
continue
json_id = self.meta_metadata[field]['json_id']
values = self.get(field)
if not isinstance(values, (list, tuple)):
values = (values, )
for value in values:
try:
rules = get_producer_rules(json_id, 'json_for_ld', 'recordext')
for rule in rules:
tags = rule[0] if isinstance(rule[0], tuple) \
else (rule[0], )
if tags and not any([
tag in tags
for tag in self.meta_metadata[field]['function']
]):
continue
tmp_dict = {}
for key, subfield in rule[1].items():
if not subfield:
tmp_dict[key] = value
else:
try:
tmp_dict[key] = value[subfield]
except:
try:
tmp_dict[key] = try_to_eval(
subfield,
functions(
self.additional_info.namespace),
value=value,
self=self)
except ImportError:
pass
except Exception as e:
self.continuable_errors.append(
"Producer CError - Unable to produce "
"'%s'.\n %s" % (field, str(e)))
if tmp_dict:
for k, v in tmp_dict.items():
if isinstance(v, list):
if k not in out:
out[k] = []
for element in v:
out[k].append(element)
else:
out[k] = v
except KeyError as e:
self.continuable_errors.append(
"Producer CError - Unable to produce '%s'"
" (No rule found).\n %s" % (field, str(e)))
return out
