def create_element(cls, rule, namespace): """Creates the dictionary with the dump and load functions""" return {'loads': try_to_eval(rule.json_ext.loads, functions(namespace)), 'dumps': try_to_eval(rule.json_ext.dumps, functions(namespace))}
def create_element(cls, rule, namespace): """Create the dictionary with the info from the configuratin file.""" element = dict() element['mapping'] = try_to_eval( rule.mapping, functions(namespace)) if rule.mapping else {} element['facets'] = try_to_eval( rule.facets, functions(namespace)) if rule.facets else {} element['highlights'] = try_to_eval( rule.highlights, functions(namespace)) if rule.highlights else {} element['local_tokenizer'] = [] if rule.local_tokenizer: for i in range(0, len(rule.local_tokenizer), 2): element['local_tokenizer'].append( (rule.local_tokenizer[i], rule.local_tokenizer[i + 1])) return element
def create(cls, data, schema=None, **kwargs): db.session.begin(subtransactions=True) try: data.setdefault('_id', str(uuid.uuid4())) data.setdefault('creation_date', datetime.now().isoformat()) document = cls(data) from invenio.modules.jsonalchemy.registry import functions list(functions('documentext')) toposort_send(signals.before_document_insert, document) if schema is not None: validate(document, schema) model = DocumentMetadata(id=document['_id'], json=dict(document)) db.session.add(model) db.session.commit() document.model = model toposort_send(signals.after_document_insert, document) return document except Exception: current_app.logger.exception("Problem creating a document.") db.session.rollback() raise
def create(cls, data, schema=None): db.session.begin(subtransactions=True) try: record = cls(unicodifier(data)) from invenio.modules.jsonalchemy.registry import functions list(functions('recordext')) toposort_send(before_record_insert, record) if schema is not None: validate(record, schema) metadata = dict(json=dict(record)) if record.get('recid', None) is not None: metadata['id'] = record.get('recid') db.session.add(RecordMetadata(**metadata)) db.session.commit() toposort_send(after_record_insert, record) return record except Exception: current_app.logger.exception("Problem creating a record.") db.session.rollback() raise
def create(cls, data, schema=None): db.session.begin(subtransactions=True) try: record = cls(unicodifier(data)) from invenio.modules.jsonalchemy.registry import functions list(functions("recordext")) toposort_send(before_record_insert, record) if schema is not None: validate(record, schema) metadata = dict(json=dict(record)) if record.get("recid", None) is not None: metadata["id"] = record.get("recid") db.session.add(RecordMetadata(**metadata)) db.session.commit() toposort_send(after_record_insert, record) return record except Exception: current_app.logger.exception("Problem creating a record.") db.session.rollback() raise
def memoize(_id, field_name): func = reduce( lambda obj, key: obj[key], json.meta_metadata[field_name]['function'], FieldParser.field_definitions(json.additional_info.namespace)) return try_to_eval(func, functions(json.additional_info.namespace), self=json)
def index_holdingpen_record(sender, **kwargs): """Index a Holding Pen record.""" from invenio.ext.es import es from invenio_records.api import Record from invenio_records.signals import before_record_insert from .registry import workflows from .models import ObjectVersion if not sender.workflow: # No workflow registered to object yet. Skip indexing return workflow = workflows.get(sender.workflow.name) if not workflow: current_app.logger.error( "No workflow found for sender: {0}".format(sender.id) ) return if not hasattr(sender, 'data'): sender.data = sender.get_data() if not hasattr(sender, 'extra_data'): sender.extra_data = sender.get_extra_data() record = Record({}) record["version"] = ObjectVersion.name_from_version(sender.version) record["type"] = sender.data_type record["status"] = sender.status record["created"] = sender.created.isoformat() record["modified"] = sender.modified.isoformat() record["uri"] = sender.uri record["id_workflow"] = sender.id_workflow record["id_user"] = sender.id_user record["id_parent"] = sender.id_parent record["workflow"] = sender.workflow.name try: record.update(workflow.get_record(sender)) except Exception as err: current_app.logger.exception(err) try: record.update(workflow.get_sort_data(sender)) except Exception as err: current_app.logger.exception(err) from invenio.modules.jsonalchemy.registry import functions list(functions('recordext')) toposort_send(before_record_insert, record) es.index( index=current_app.config.get("WORKFLOWS_HOLDING_PEN_INDEX"), doc_type='record', body=dict(record), id=sender.id )
def _apply_rules(self, json_id, field_name, rule): """Override default behavior. See :meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules`. Tries to apply all the rules defined for marc and as soon as it can apply one it stops the process. It also keeps the marc_tag that comes from :meth:`._get_elements_from_blob` :param json_id: as in :meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules` :param field_name: as in :meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules` :param rule: as in :meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules` """ for field_def in rule['rules'].get( self._json.additional_info.master_format, []): if not self._evaluate_before_decorators(field_def): continue marc_tag, elements = self._get_elements_from_blob( field_def['source_tags']) if not isinstance(elements, (list, tuple)): elements = (elements, ) for element in elements: tmp_field_def = copy.deepcopy(field_def) tmp_field_def['source_tags'] = [ marc_tag, ] if not self._evaluate_on_decorators(tmp_field_def, element): continue try: value = try_to_eval( tmp_field_def['function'], functions(self._json.additional_info.namespace), value=element, self=self._json) self._remove_none_values(value) info = self._find_field_metadata(json_id, field_name, 'creator', tmp_field_def) self._json['__meta_metadata__'][field_name] = info self._json.__setitem__( field_name, value, extend=True, exclude=['decorators', 'extensions']) except Exception as e: self._json.errors.append( "Rule Error - Unable to apply rule for field " "'%s' with value '%s'. \n%s" % (field_name, element, str(e)), ) if field_name in self._json._dict_bson: break
def evaluate(cls, value, namespace, args): """Evaluate ``args`` with the master value from the input. :returns: a boolean depending on evaluated ``value``. """ from invenio.modules.jsonalchemy.registry import functions evaluated = try_to_eval(args, functions(namespace), value=value) if not isinstance(evaluated, (list, tuple)): return evaluated else: return all(evaluated)
def evaluate(cls, reader, args): """ This is a special case where the real evaluation of the decorator happened before the evaluation, :see:create_element """ from invenio.modules.jsonalchemy.registry import functions evaluated = try_to_eval(args, functions(reader._json.additional_info.namespace), self=reader._json) if not isinstance(evaluated, (list, tuple)): return evaluated else: return all(evaluated)
def _apply_rules(self, json_id, field_name, rule): """Override default behavior. See :meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules`. Tries to apply all the rules defined for marc and as soon as it can apply one it stops the process. It also keeps the marc_tag that comes from :meth:`._get_elements_from_blob` :param json_id: as in :meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules` :param field_name: as in :meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules` :param rule: as in :meth:`~invenio.modules.jsonalchemy.readers.Reader._apply_rules` """ for field_def in rule['rules'].get( self._json.additional_info.master_format, []): if not self._evaluate_before_decorators(field_def): continue marc_tag, elements = self._get_elements_from_blob( field_def['source_tags']) if not isinstance(elements, (list, tuple)): elements = (elements, ) for element in elements: tmp_field_def = copy.deepcopy(field_def) tmp_field_def['source_tags'] = [marc_tag, ] if not self._evaluate_on_decorators(tmp_field_def, element): continue try: value = try_to_eval( tmp_field_def['function'], functions(self._json.additional_info.namespace), value=element, self=self._json) self._remove_none_values(value) info = self._find_field_metadata(json_id, field_name, 'creator', tmp_field_def) self._json['__meta_metadata__'][field_name] = info self._json.__setitem__(field_name, value, extend=True, exclude=['decorators', 'extensions']) except Exception as e: self._json.errors.append( "Rule Error - Unable to apply rule for field " "'%s' with value '%s'. \n%s" % (field_name, element, str(e)),) if field_name in self._json._dict_bson: break
def evaluate(cls, reader, args): """Evaluate parser. This is a special case where the real evaluation of the decorator is happening before the evaluation. """ from invenio.modules.jsonalchemy.registry import functions evaluated = try_to_eval(args, functions( reader._json.additional_info.namespace), self=reader._json) if not isinstance(evaluated, (list, tuple)): return evaluated else: return all(evaluated)
def evaluate(cls, json, field_name, action, args): """ Applies the connect funtion with json, field_name and action parameters if any functions availabe, otherwise it will put the content of the current field into the connected one. """ if action == 'get': return from invenio.modules.jsonalchemy.registry import functions for info in args: if info['update_function'] is None: json.__setitem__(info['connected_field'], json[field_name], exclude='connect') else: try_to_eval(info['update_function'], functions(json.additional_info.namespace))( json, field_name, info['connected_field'], action)
def evaluate(cls, json, field_name, action, args): """Evaluate the parser. When getting a json field compare the timestamp and the lifetime of it and, if it the lifetime is over calculate its value again. If the value of the field has changed since the last time it gets updated in the DB. """ if cls.__cache is None: cls.__cache = import_string( cfg.get('CFG_JSONALCHEMY_CACHE', 'invenio.ext.cache:cache')) @cls.__cache.memoize(timeout=args) def memoize(_id, field_name): func = reduce( lambda obj, key: obj[key], json.meta_metadata[field_name]['function'], FieldParser.field_definitions(json.additional_info.namespace)) return try_to_eval(func, functions(json.additional_info.namespace), self=json) if args == cls.DEFAULT_TIMEOUT: return if action == 'get': if args == 0: # No cached version is stored, retrieve it func = reduce( lambda obj, key: obj[key], json.meta_metadata[field_name]['function'], FieldParser.field_definitions( json.additional_info.namespace)) json._dict_bson[field_name] = try_to_eval( func, functions(json.additional_info.namespace), self=json) else: json._dict_bson[field_name] = memoize(json.get('_id'), field_name) elif action == 'set': if args >= 0: # Don't store anything json._dict_bson[field_name] = None
def produce(self, fields=None): """Export the json in marc format. Produces a list of dictionaries will all the possible marc tags as keys. :param fields: list of fields to include in the output, if None or empty list all available tags will be included. """ from invenio.base.utils import try_to_eval from invenio.modules.jsonalchemy.parser import get_producer_rules from invenio.modules.jsonalchemy.registry import functions if not fields: fields = self.keys() out = [] for field in fields: if field.startswith('__') or self.get(field) is None: continue json_id = self.meta_metadata[field]['json_id'] values = self.get(field) if not isinstance(values, (list, tuple)): values = (values, ) for value in values: try: for rule in get_producer_rules( json_id, 'json_for_marc', self.additional_info['namespace']): marc_tags = rule[0] if isinstance(rule[0], tuple) \ else (rule[0], ) if marc_tags and not any([ re.match(m, t) for m in marc_tags for t in self.meta_metadata[field]['function'] ]): # Not match, continue to next rule continue tmp_dict = dict() for marc_tag, subfield in iteritems(rule[1]): if len(marc_tag) == 1: marc_tag = \ self.meta_metadata[field]['function'][0] + \ marc_tag if not subfield: tmp_dict[marc_tag] = value else: try: tmp_dict[marc_tag] = value[subfield] except: try: # Evaluate only non keyword values. if subfield in __builtins__: raise ImportError tmp_dict[marc_tag] = try_to_eval( subfield, functions( self.additional_info.namespace), value=value, self=self) except ImportError: pass except Exception as e: self.continuable_errors.append( "Producer CError - Unable to produce " "'%s'.\n %s" % (field, str(e))) if tmp_dict: out.append(tmp_dict) except Exception as e: self.continuable_errors.append( "Producer CError - Unable to produce '%s'.\n %s" % (field, str(e))) return out
def create_element(cls, rule, namespace): """Just evaluate the content of the schema to a python dictionary.""" return try_to_eval(rule.schema, functions(namespace))
def create_element(cls, rule, namespace): """Create the dictionary with the dump and load functions.""" return { 'loads': try_to_eval(rule.json_ext.loads, functions(namespace)), 'dumps': try_to_eval(rule.json_ext.dumps, functions(namespace)) }
def produce(self, fields=None): """Export the json in marc format. Produces a list of dictionaries will all the possible marc tags as keys. :param fields: list of fields to include in the output, if None or empty list all available tags will be included. """ from invenio.base.utils import try_to_eval from invenio.modules.jsonalchemy.parser import get_producer_rules from invenio.modules.jsonalchemy.registry import functions if not fields: fields = self.keys() out = [] for field in fields: if field.startswith('__') or self.get(field) is None: continue json_id = self.meta_metadata[field]['json_id'] values = self.get(field) if not isinstance(values, (list, tuple)): values = (values, ) for value in values: try: for rule in get_producer_rules( json_id, 'json_for_marc', self.additional_info['namespace']): marc_tags = rule[0] if isinstance(rule[0], tuple) \ else (rule[0], ) if marc_tags and not any( [re.match(m, t) for m in marc_tags for t in self.meta_metadata[field]['function']]): # Not match, continue to next rule continue tmp_dict = dict() for marc_tag, subfield in iteritems(rule[1]): if len(marc_tag) == 1: marc_tag = \ self.meta_metadata[field]['function'][0] + \ marc_tag if not subfield: tmp_dict[marc_tag] = value else: try: tmp_dict[marc_tag] = value[subfield] except: try: # Evaluate only non keyword values. if subfield in __builtins__: raise ImportError tmp_dict[marc_tag] = try_to_eval( subfield, functions( self.additional_info.namespace ), value=value, self=self) except ImportError: pass except Exception as e: self.continuable_errors.append( "Producer CError - Unable to produce " "'%s'.\n %s" % (field, str(e))) if tmp_dict: out.append(tmp_dict) except Exception as e: self.continuable_errors.append( "Producer CError - Unable to produce '%s'.\n %s" % (field, str(e))) return out
def produce(self, fields=None): """Export the json for indexing purposes. :param fields: list of fields to include in the output, if None or empty list all available tags will be included. """ from invenio.base.utils import try_to_eval from invenio.modules.jsonalchemy.parser import get_producer_rules from invenio.modules.jsonalchemy.registry import functions out = self.dumps(without_meta_metadata=True, with_calculated_fields=True, keywords=fields) for field, values in iteritems(out): if field.startswith('__'): continue json_id = self.meta_metadata[field]['json_id'] tmp_dict = dict() if not isinstance(values, (list, tuple)): values = (values, ) for value in values: try: for rule in get_producer_rules( json_id, 'json_for_indexer', self.additional_info['namespace']): # FIXME add support of indexer names. # indexer_names = rule[0] if isinstance(rule[0], tuple) \ # else (rule[0], ) # if indexer_names and not any( # [m == cfg['INDEXER_ENGINE']) # for m in indexer_names]) # # Not match, continue to next rule # continue for subfield, value_or_function in iteritems(rule[1]): try: # Evaluate only non keyword values. if value_or_function in __builtins__: raise ImportError tmp_dict[subfield] = try_to_eval( value_or_function, functions( self.additional_info.namespace ), value=value, self=self) except ImportError: pass except Exception as e: self.continuable_errors.append( "Producer CError - Unable to produce " "'%s'.\n %s" % (field, str(e))) if tmp_dict: if value is None: value = tmp_dict elif isinstance(value, dict): value.update(tmp_dict) else: raise RuntimeError("Invalid field structure.") except Exception as e: self.continuable_errors.append( "Producer CError - Unable to produce '%s'.\n %s" % (field, str(e))) return out
def produce(self, fields=None): """Export the record in JSON-LD format. @param tags: list of tags to include in the output, if None or empty list all available tags will be included. """ from invenio.base.utils import try_to_eval from invenio.modules.jsonalchemy.parser import get_producer_rules from invenio.modules.jsonalchemy.registry import functions if not fields: fields = self.keys() out = {} for field in fields: if field.startswith('__') or self.get(field) is None: continue json_id = self.meta_metadata[field]['json_id'] values = self.get(field) if not isinstance(values, (list, tuple)): values = (values, ) for value in values: try: rules = get_producer_rules(json_id, 'json_for_ld', 'recordext') for rule in rules: tags = rule[0] if isinstance(rule[0], tuple) \ else (rule[0], ) if tags and not any([ tag in tags for tag in self.meta_metadata[field]['function']]): continue tmp_dict = {} for key, subfield in rule[1].items(): if not subfield: tmp_dict[key] = value else: try: tmp_dict[key] = value[subfield] except: try: tmp_dict[key] = try_to_eval( subfield, functions( self.additional_info.namespace ), value=value, self=self) except ImportError: pass except Exception as e: self.continuable_errors.append( "Producer CError - Unable to produce " "'%s'.\n %s" % (field, str(e))) if tmp_dict: for k, v in tmp_dict.items(): if isinstance(v, list): if k not in out: out[k] = [] for element in v: out[k].append(element) else: out[k] = v except KeyError as e: self.continuable_errors.append( "Producer CError - Unable to produce '%s'" " (No rule found).\n %s" % (field, str(e))) return out
def produce(self, fields=None): """Export the record in JSON-LD format. @param tags: list of tags to include in the output, if None or empty list all available tags will be included. """ from invenio.base.utils import try_to_eval from invenio.modules.jsonalchemy.parser import get_producer_rules from invenio.modules.jsonalchemy.registry import functions if not fields: fields = self.keys() out = {} for field in fields: if field.startswith('__') or self.get(field) is None: continue json_id = self.meta_metadata[field]['json_id'] values = self.get(field) if not isinstance(values, (list, tuple)): values = (values, ) for value in values: try: rules = get_producer_rules(json_id, 'json_for_ld', 'recordext') for rule in rules: tags = rule[0] if isinstance(rule[0], tuple) \ else (rule[0], ) if tags and not any([ tag in tags for tag in self.meta_metadata[field]['function'] ]): continue tmp_dict = {} for key, subfield in rule[1].items(): if not subfield: tmp_dict[key] = value else: try: tmp_dict[key] = value[subfield] except: try: tmp_dict[key] = try_to_eval( subfield, functions( self.additional_info.namespace), value=value, self=self) except ImportError: pass except Exception as e: self.continuable_errors.append( "Producer CError - Unable to produce " "'%s'.\n %s" % (field, str(e))) if tmp_dict: for k, v in tmp_dict.items(): if isinstance(v, list): if k not in out: out[k] = [] for element in v: out[k].append(element) else: out[k] = v except KeyError as e: self.continuable_errors.append( "Producer CError - Unable to produce '%s'" " (No rule found).\n %s" % (field, str(e))) return out