def memoize(_id, field_name): func = reduce( lambda obj, key: obj[key], json.meta_metadata[field_name]['function'], FieldParser.field_definitions(json.additional_info.namespace)) return try_to_eval(func, functions(json.additional_info.namespace), self=json)
def get_records_facets_config(): """Get facets config for records.""" from invenio.modules.jsonalchemy.parser import FieldParser fields = FieldParser.field_definitions('recordext') facets = {} for name, value in fields.iteritems(): current_facet = value.get("elasticsearch", {}).get("facets") if current_facet: facets.update(current_facet) return facets
def get_records_fields_config(): """Mapping for records.""" from invenio.modules.jsonalchemy.parser import FieldParser fields = FieldParser.field_definitions('recordext') mapping = {} for name, value in fields.iteritems(): current_mapping = value.get("elasticsearch", {}).get("mapping") if current_mapping: mapping.update(current_mapping) return mapping
def evaluate(cls, json, field_name, action, args): """Evaluate the dumps and loads functions depending on the action.""" from invenio.modules.jsonalchemy.parser import FieldParser if action == 'set': try: json._dict[field_name] = reduce( lambda obj, key: obj[key], args['dumps'], FieldParser.field_definitions( json.additional_info.namespace))( json._dict_bson[field_name]) except (KeyError, IndexError, TypeError): json._dict[field_name] = json._dict_bson[field_name] elif action == 'get': try: json._dict_bson[field_name] = reduce( lambda obj, key: obj[key], args['loads'], FieldParser.field_definitions( json.additional_info.namespace))( json._dict[field_name]) except (KeyError, IndexError, TypeError): json._dict_bson[field_name] = json._dict[field_name]
def evaluate(cls, json, field_name, action, args): """Evaluate the dumps and loads functions depending on the action""" from invenio.modules.jsonalchemy.parser import FieldParser if action == 'set': try: json._dict[field_name] = reduce( lambda obj, key: obj[key], args['dumps'], FieldParser.field_definitions( json.additional_info.namespace))(json._dict_bson[field_name]) except (KeyError, IndexError, TypeError, AttributeError): json._dict[field_name] = json._dict_bson[field_name] elif action == 'get': try: json._dict_bson[field_name] = reduce( lambda obj, key: obj[key], args['loads'], FieldParser.field_definitions( json.additional_info.namespace))(json._dict[field_name]) except (KeyError, IndexError, TypeError): json._dict_bson[field_name] = json._dict[field_name]
def create_element(cls, rule, namespace): """Prepare the list of producers with their names and parameters.""" id_ = rule.field['json_id'] producers = {} if not rule.extend else \ FieldParser.field_definitions(namespace)[id_].get('producer', {}) for producer in rule.producer: if producer['code'] not in producers: producers[producer['code']] = [] producers[producer['code']].append( (producer['params'], producer['rule'])) return producers
def create_element(cls, rule, namespace): """Prepare the list of producers with their names and parameters.""" id_ = rule.field['json_id'] producers = {} if not rule.extend else \ FieldParser.field_definitions(namespace)[id_].get('producer', {}) for producer in rule.producer: if producer['code'] not in producers: producers[producer['code']] = [] producers[producer['code']].append((producer['params'], producer['rule'])) return producers
def create_element(cls, rule, namespace): """Prepare the list of aggregations with their names and parameters.""" id_ = rule.field['json_id'] aggregations = {} if not rule.extend else \ FieldParser.field_definitions(namespace)[id_].get( 'aggregation', {}) for aggregation in rule.aggregation: if aggregation['name'] not in aggregations: aggregations[aggregation['name']] = [] aggregations[aggregation['name']].append(( aggregation['engine'], aggregation['rule'])) return aggregations
def get_records_highlights_config(): """Get hilights config for records.""" from invenio.modules.jsonalchemy.parser import FieldParser fields = FieldParser.field_definitions('recordext') highlights = {} for name, value in fields.iteritems(): current_highlights = value.get("elasticsearch", {}).get("highlights") if current_highlights: highlights.update(current_highlights) config = { "fields": highlights } return config
def create_element(cls, rule, namespace): """Prepare the list of aggregations with their names and parameters.""" id_ = rule.field['json_id'] aggregations = {} if not rule.extend else \ FieldParser.field_definitions(namespace)[id_].get( 'aggregation', {}) for aggregation in rule.aggregation: if aggregation['name'] not in aggregations: aggregations[aggregation['name']] = [] aggregations[aggregation['name']].append( (aggregation['engine'], aggregation['rule'])) return aggregations
def jsonschema(namespace, model): from invenio.modules.jsonalchemy.parser import ModelParser from invenio.modules.jsonalchemy.parser import FieldParser site_url = current_app.config['CFG_SITE_URL'] out = { "$schema": "http://json-schema.org/schema#", "id": "{site_url}/schemas/{model}-0.0.1.json".format(site_url=site_url, model=model), "type": "object", "properties": {}, "required": [] } fields = ModelParser.resolve_models(model, namespace).get('fields', {}) out['required'] = fields.keys() for field in fields: parsed_field = FieldParser.field_definition_model_based( field, model, namespace) description = parsed_field.pop('description', None) sub_properties = {} try: for sub_property in parsed_field['producer']['json_for_marc'][0][ 1].values(): sub_properties[sub_property] = {'type': 'string'} except (KeyError, IndexError): pass property_ = out['properties'][field] = { 'type': 'array', 'uniqueItems': True, 'items': { 'type': 'object', 'properties': sub_properties }, 'other': str(parsed_field), } if description: property_['description'] = description print(json.dumps(out, sort_keys=True))
def evaluate(cls, json, field_name, action, args): """Evaluate the parser. When getting a json field compare the timestamp and the lifetime of it and, if it the lifetime is over calculate its value again. If the value of the field has changed since the last time it gets updated in the DB. """ if cls.__cache is None: cls.__cache = import_string( cfg.get('CFG_JSONALCHEMY_CACHE', 'invenio.ext.cache:cache')) @cls.__cache.memoize(timeout=args) def memoize(_id, field_name): func = reduce( lambda obj, key: obj[key], json.meta_metadata[field_name]['function'], FieldParser.field_definitions(json.additional_info.namespace)) return try_to_eval(func, functions(json.additional_info.namespace), self=json) if args == cls.DEFAULT_TIMEOUT: return if action == 'get': if args == 0: # No cached version is stored, retrieve it func = reduce( lambda obj, key: obj[key], json.meta_metadata[field_name]['function'], FieldParser.field_definitions( json.additional_info.namespace)) json._dict_bson[field_name] = try_to_eval( func, functions(json.additional_info.namespace), self=json) else: json._dict_bson[field_name] = memoize(json.get('_id'), field_name) elif action == 'set': if args >= 0: # Don't store anything json._dict_bson[field_name] = None
def jsonschema(namespace, model): from invenio.modules.jsonalchemy.parser import ModelParser from invenio.modules.jsonalchemy.parser import FieldParser site_url = current_app.config['CFG_SITE_URL'] out = {"$schema": "http://json-schema.org/schema#", "id": "{site_url}/schemas/{model}-0.0.1.json".format( site_url=site_url, model=model ), "type": "object", "properties": {}, "required": []} fields = ModelParser.resolve_models(model, namespace).get('fields', {}) out['required'] = fields.keys() for field in fields: parsed_field = FieldParser.field_definition_model_based(field, model, namespace) description = parsed_field.pop('description', None) sub_properties = {} try: for sub_property in parsed_field['producer'][ 'json_for_marc'][0][1].values(): sub_properties[sub_property] = {'type': 'string'} except (KeyError, IndexError): pass property_ = out['properties'][field] = { 'type': 'array', 'uniqueItems': True, 'items': {'type': 'object', 'properties': sub_properties}, 'other': str(parsed_field), } if description: property_['description'] = description print(json.dumps(out, sort_keys=True))
def create_element(cls, rule, field_def, content, namespace): """Simply returns the list with the tuples""" from invenio.modules.jsonalchemy.parser import FieldParser if isinstance(content, dict): content = (content, ) else: content = content.asList() # Conect to the other side for connect in content: try: connected_field = FieldParser.field_definitions(namespace)[ connect['connected_field']] except KeyError: raise FieldParserException( "Definition for '%(field)s' not found, maybe adding " "@parse_first('%(field)s') could help" % {'field': connect['connected_field']}) # Add it to all the rules (all master format, derived and # calculated) for connected_rules in connected_field['rules'].values(): for connected_rule in connected_rules: # Add parse_first for connected field if 'parse_first' not in connected_rule['decorators'][ 'before']: connected_rule['decorators']['before'][ 'parse_first'] = [] connected_rule['decorators']['before']['parse_first']\ .append(rule.field['json_id']) # Connect fields if 'connect' not in connected_rule['decorators']['after']: connected_rule['decorators']['after']['connect'] = [] connected_rule['decorators']['after']['connect']\ .append({'connected_field': rule.field['json_id'], 'update_function': connect['update_function'] }) return content
def create_element(cls, rule, field_def, content, namespace): """Simply returns the list with the tuples""" from invenio.modules.jsonalchemy.parser import FieldParser if isinstance(content, dict): content = (content, ) else: content = content.asList() # Conect to the other side for connect in content: try: connected_field = FieldParser.field_definitions( namespace)[connect['connected_field']] except KeyError: raise FieldParserException( "Definition for '%(field)s' not found, maybe adding " "@parse_first('%(field)s') could help" % {'field': connect['connected_field']}) # Add it to all the rules (all master format, derived and # calculated) for connected_rules in connected_field['rules'].values(): for connected_rule in connected_rules: # Add parse_first for connected field if 'parse_first' not in connected_rule['decorators'][ 'before']: connected_rule['decorators']['before'][ 'parse_first'] = [] connected_rule['decorators']['before']['parse_first']\ .append(rule.field['json_id']) # Connect fields if 'connect' not in connected_rule['decorators']['after']: connected_rule['decorators']['after']['connect'] = [] connected_rule['decorators']['after']['connect']\ .append({'connected_field': rule.field['json_id'], 'update_function': connect['update_function'] }) return content