def test_field_name_model_based(self):
        """JSONAlchemy - field name model based"""
        field_model_def = self.model_parser.field_definition_model_based(
            'title', 'test_model')
        field_def = self.field_parser.field_definitions()['title_title']

        value = {'a': 'Awesome title', 'b': 'sub title', 'k': 'form'}
        from jsonalchemy.utils import try_to_eval

        self.assertEqual(
            try_to_eval(field_model_def['rules'][
                        'marc'][0]['function'], value=value),
            try_to_eval(field_def['rules']['marc'][0]['function'],
                        value=value))
Beispiel #2
0
    def test_field_name_model_based(self):
        """JSONAlchemy - field name model based"""
        field_model_def = self.model_parser.field_definition_model_based(
            'title', 'test_model')
        field_def = self.field_parser.field_definitions()['title_title']

        value = {'a': 'Awesome title', 'b': 'sub title', 'k': 'form'}
        from jsonalchemy.utils import try_to_eval

        self.assertEqual(
            try_to_eval(field_model_def['rules']['marc'][0]['function'],
                        value=value),
            try_to_eval(field_def['rules']['marc'][0]['function'],
                        value=value))
Beispiel #3
0
    def _apply_virtual_rules(self, json_id, field_name, rule):
        """Try to apply either a 'derived' or 'calculated' rule.

        :param json_id: Name os the json field in the configuration file.
        :param field_name: Final name of the field, taken from the model
            definiti, if any, otherwise is equal to the `json_id`
        :param rule: Current rule for the `json_id`
        """
        field_defs = []
        field_defs.append(('calculated', rule['rules'].get('calculated', [])))
        field_defs.append(('derived', rule['rules'].get('derived', [])))
        for (field_type, _field_def) in field_defs:
            for field_def in _field_def:
                if not self._evaluate_before_decorators(field_def):
                    continue
                try:
                    value = try_to_eval(
                        field_def['function'],
                        self.metadata.functions,
                        self=self._json)
                    self._remove_none_values(value)
                    info = self._find_field_metadata(json_id, field_name,
                                                     field_type, field_def)
                    self._json['__meta_metadata__'][field_name] = info
                    self._json.__setitem__(
                        field_name, value, extend=False,
                        exclude=['decorators', 'extensions'])
                except Exception as e:
                    self._json.errors.append(
                        "Rule Error - Unable to apply rule for virtual "
                        "field '%s'. \n%s" % (field_name, str(e)),)
Beispiel #4
0
    def _apply_virtual_rules(self, json_id, field_name, rule):
        """Try to apply either a 'derived' or 'calculated' rule.

        :param json_id: Name os the json field in the configuration file.
        :param field_name: Final name of the field, taken from the model
            definiti, if any, otherwise is equal to the `json_id`
        :param rule: Current rule for the `json_id`
        """
        field_defs = []
        field_defs.append(('calculated', rule['rules'].get('calculated', [])))
        field_defs.append(('derived', rule['rules'].get('derived', [])))
        for (field_type, _field_def) in field_defs:
            for field_def in _field_def:
                if not self._evaluate_before_decorators(field_def):
                    continue
                try:
                    value = try_to_eval(field_def['function'],
                                        self.metadata.functions,
                                        self=self._json)
                    self._remove_none_values(value)
                    info = self._find_field_metadata(json_id, field_name,
                                                     field_type, field_def)
                    self._json['__meta_metadata__'][field_name] = info
                    self._json.__setitem__(
                        field_name,
                        value,
                        extend=False,
                        exclude=['decorators', 'extensions'])
                except Exception as e:
                    self._json.errors.append(
                        "Rule Error - Unable to apply rule for virtual "
                        "field '%s'. \n%s" % (field_name, str(e)), )
    def evaluate(cls, json, field_name, action, args):
        """
        Applies the connect funtion with json, field_name and action parameters
        if any functions availabe, otherwise it will put the content of the
        current field into the connected one.
        """
        if action == 'get':
            return

        for info in args:
            if info['update_function'] is None:
                json.__setitem__(info['connected_field'],
                                 json[field_name],
                                 exclude='connect')
            else:
                try_to_eval(info['update_function'], json.metadata.functions)(
                    json, field_name, info['connected_field'], action)
Beispiel #6
0
    def evaluate(cls, value, metadata, args):
        """Evaluate ``args`` with the master value from the input.

        :returns: a boolean depending on evaluated ``value``.
        """
        evaluated = try_to_eval(args, metadata.functions, value=value)
        if not isinstance(evaluated, (list, tuple)):
            return evaluated
        else:
            return all(evaluated)
Beispiel #7
0
    def evaluate(cls, json, field_name, action, args):
        """
        Applies the connect funtion with json, field_name and action parameters
        if any functions availabe, otherwise it will put the content of the
        current field into the connected one.
        """
        if action == 'get':
            return

        for info in args:
            if info['update_function'] is None:
                json.__setitem__(info['connected_field'],
                                 json[field_name],
                                 exclude='connect')
            else:
                try_to_eval(info['update_function'],
                            json.metadata.functions)(json, field_name,
                                                     info['connected_field'],
                                                     action)
Beispiel #8
0
    def _apply_rules(self, json_id, field_name, rule):
        """Override default behavior.

        See :meth:`~jsonalchemy.readers.Reader._apply_rules`.

        Tries to apply all the rules defined for marc and as soon as it can
        apply one it stops the process.
        It also keeps the marc_tag that comes from
        :meth:`._get_elements_from_blob`

        :param json_id: as in
            :meth:`~jsonalchemy.readers.Reader._apply_rules`
        :param field_name: as in
            :meth:`~jsonalchemy.readers.Reader._apply_rules`
        :param rule: as in
            :meth:`~jsonalchemy.readers.Reader._apply_rules`
        """
        for field_def in rule['rules'].get(
                self._json.additional_info.master_format, []):
            if not self._evaluate_before_decorators(field_def):
                continue
            marc_tag, elements = self._get_elements_from_blob(
                field_def['source_tags'])
            if not isinstance(elements, (list, tuple)):
                elements = (elements, )
            for element in elements:
                tmp_field_def = copy.deepcopy(field_def)
                tmp_field_def['source_tags'] = [
                    marc_tag,
                ]
                if not self._evaluate_on_decorators(tmp_field_def, element):
                    continue
                try:
                    value = try_to_eval(tmp_field_def['function'],
                                        self.metadata.functions,
                                        value=element,
                                        self=self._json)
                    self._remove_none_values(value)
                    info = self._find_field_metadata(json_id, field_name,
                                                     'creator', tmp_field_def)
                    self._json['__meta_metadata__'][field_name] = info
                    self._json.__setitem__(
                        field_name,
                        value,
                        extend=True,
                        exclude=['decorators', 'extensions'])
                except Exception as e:
                    self._json.errors.append(
                        "Rule Error - Unable to apply rule for field "
                        "'%s' with value '%s'. \n%s" %
                        (field_name, element, str(e)), )
            if field_name in self._json._dict_bson:
                break
Beispiel #9
0
    def evaluate(cls, reader, args):
        """Evaluate parser.

        This is a special case where the real evaluation of the decorator
        is happening before the evaluation.
        """
        evaluated = try_to_eval(args,
                                reader.metadata.functions,
                                self=reader._json)
        if not isinstance(evaluated, (list, tuple)):
            return evaluated
        else:
            return all(evaluated)
    def evaluate(cls, reader, args):
        """Evaluate parser.

        This is a special case where the real evaluation of the decorator
        is happening before the evaluation.
        """
        evaluated = try_to_eval(
            args, reader.metadata.functions,
            self=reader._json)
        if not isinstance(evaluated, (list, tuple)):
            return evaluated
        else:
            return all(evaluated)
Beispiel #11
0
    def _apply_rules(self, json_id, field_name, rule):
        """Override default behavior.

        See :meth:`~jsonalchemy.readers.Reader._apply_rules`.

        Tries to apply all the rules defined for marc and as soon as it can
        apply one it stops the process.
        It also keeps the marc_tag that comes from
        :meth:`._get_elements_from_blob`

        :param json_id: as in
            :meth:`~jsonalchemy.readers.Reader._apply_rules`
        :param field_name: as in
            :meth:`~jsonalchemy.readers.Reader._apply_rules`
        :param rule: as in
            :meth:`~jsonalchemy.readers.Reader._apply_rules`
        """
        for field_def in rule['rules'].get(
                self._json.additional_info.master_format, []):
            if not self._evaluate_before_decorators(field_def):
                continue
            marc_tag, elements = self._get_elements_from_blob(
                field_def['source_tags'])
            if not isinstance(elements, (list, tuple)):
                elements = (elements, )
            for element in elements:
                tmp_field_def = copy.deepcopy(field_def)
                tmp_field_def['source_tags'] = [marc_tag, ]
                if not self._evaluate_on_decorators(tmp_field_def, element):
                    continue
                try:
                    value = try_to_eval(
                        tmp_field_def['function'],
                        self.metadata.functions,
                        value=element, self=self._json)
                    self._remove_none_values(value)
                    info = self._find_field_metadata(json_id, field_name,
                                                     'creator', tmp_field_def)
                    self._json['__meta_metadata__'][field_name] = info
                    self._json.__setitem__(field_name, value, extend=True,
                                           exclude=['decorators',
                                                    'extensions'])
                except Exception as e:
                    self._json.errors.append(
                        "Rule Error - Unable to apply rule for field "
                        "'%s' with value '%s'. \n%s"
                        % (field_name, element, str(e)),)
            if field_name in self._json._dict_bson:
                break
Beispiel #12
0
    def evaluate(cls, obj, args):
        """Extend the incoming object with all the new things from args."""
        if args is None:
            return
        extensions = []
        for ext in args:
            try:
                if ':' in ext:
                    package, attr = ext.split(':')
                else:
                    package, attr = ext.rsplit('.', 1)
                extensions.append(
                    getattr(importlib.import_module(package), attr))
            except (ImportError, AttributeError, KeyError):
                extensions.append(try_to_eval(ext))
        extensions.append(obj.__class__)

        obj.__class__ = type(obj.__class__.__name__, tuple(extensions), {})
    def evaluate(cls, obj, args):
        """Extend the incoming object with all the new things from args."""
        if args is None:
            return
        extensions = []
        for ext in args:
            try:
                if ':' in ext:
                    package, attr = ext.split(':')
                else:
                    package, attr = ext.rsplit('.', 1)
                extensions.append(
                    getattr(importlib.import_module(package), attr)
                )
            except (ImportError, AttributeError, KeyError):
                extensions.append(try_to_eval(ext))
        extensions.append(obj.__class__)

        obj.__class__ = type(obj.__class__.__name__, tuple(extensions), {})
Beispiel #14
0
    def _apply_rules(self, json_id, field_name, rule):
        """Try to apply a 'creator' rule.

        :param json_id: Name os the json field in the configuration file.
        :param field_name: Final name of the field, taken from the model
            definiti, if any, otherwise is equal to the `json_id`
        :param rule: Current rule for the `json_id`
        """
        for field_def in rule['rules'].get(
                self._json.additional_info.master_format, []):
            if not self._evaluate_before_decorators(field_def):
                continue
            for elements in \
                    self._get_elements_from_blob(field_def['source_tags']):
                if not isinstance(elements, (list, tuple)):
                    elements = (elements, )
                for element in elements:
                    if not self._evaluate_on_decorators(field_def, element):
                        continue
                    try:
                        value = try_to_eval(field_def['function'],
                                            self.metadata.functions,
                                            value=element,
                                            self=self._json)
                        self._remove_none_values(value)
                        info = self._find_field_metadata(
                            json_id, field_name, 'creator', field_def)
                        self._json['__meta_metadata__'][field_name] = info
                        self._json.__setitem__(
                            field_name,
                            value,
                            extend=True,
                            exclude=['decorators', 'extensions'])
                    except Exception as e:
                        self._json.errors.append(
                            "Rule Error - Unable to apply rule for field "
                            "'%s' with value '%s'. \n%s" %
                            (field_name, element, str(e)), )
    def evaluate(cls, json, field_name, action, args):
        """Evaluate the parser.

        When getting a json field compare the timestamp and the lifetime of it
        and, if it the lifetime is over calculate its value again.

        If the value of the field has changed since the last time it gets
        updated in the DB.
        """
        # if cls.__cache is None:
        #     cls.__cache = import_string(cfg.get('CFG_JSONALCHEMY_CACHE',
        #                                         'invenio.ext.cache:cache'))
        #
        # @cls.__cache.memoize(timeout=args)
        def memoize(_id, field_name):
            func = reduce(lambda obj, key: obj[key],
                          json.meta_metadata[field_name]['function'],
                          json.metadata.field_parser.field_definitions())
            return try_to_eval(func, json.metadata.functions, self=json)

        if args == cls.DEFAULT_TIMEOUT:
            return
        if action == 'get':
            if args == 0:  # No cached version is stored, retrieve it
                func = reduce(lambda obj, key: obj[key],
                              json.meta_metadata[field_name]['function'],
                              json.metadata.field_parser.field_definitions())
                json._dict_bson[field_name] = try_to_eval(
                    func,
                    json.metadata.functions,
                    self=json)
            else:
                json._dict_bson[field_name] = memoize(json.get('_id'),
                                                      field_name)
        elif action == 'set':
            if args >= 0:  # Don't store anything
                json._dict_bson[field_name] = None
Beispiel #16
0
    def _apply_rules(self, json_id, field_name, rule):
        """Try to apply a 'creator' rule.

        :param json_id: Name os the json field in the configuration file.
        :param field_name: Final name of the field, taken from the model
            definiti, if any, otherwise is equal to the `json_id`
        :param rule: Current rule for the `json_id`
        """
        for field_def in rule['rules'].get(
                self._json.additional_info.master_format, []):
            if not self._evaluate_before_decorators(field_def):
                continue
            for elements in \
                    self._get_elements_from_blob(field_def['source_tags']):
                if not isinstance(elements, (list, tuple)):
                    elements = (elements, )
                for element in elements:
                    if not self._evaluate_on_decorators(field_def, element):
                        continue
                    try:
                        value = try_to_eval(
                            field_def['function'],
                            self.metadata.functions,
                            value=element, self=self._json)
                        self._remove_none_values(value)
                        info = self._find_field_metadata(json_id, field_name,
                                                         'creator', field_def)
                        self._json['__meta_metadata__'][field_name] = info
                        self._json.__setitem__(field_name, value, extend=True,
                                               exclude=['decorators',
                                                        'extensions'])
                    except Exception as e:
                        self._json.errors.append(
                            "Rule Error - Unable to apply rule for field "
                            "'%s' with value '%s'. \n%s"
                            % (field_name, element, str(e)),)
Beispiel #17
0
    def evaluate(cls, json, field_name, action, args):
        """Evaluate the parser.

        When getting a json field compare the timestamp and the lifetime of it
        and, if it the lifetime is over calculate its value again.

        If the value of the field has changed since the last time it gets
        updated in the DB.
        """

        # if cls.__cache is None:
        #     cls.__cache = import_string(cfg.get('CFG_JSONALCHEMY_CACHE',
        #                                         'invenio.ext.cache:cache'))
        #
        # @cls.__cache.memoize(timeout=args)
        def memoize(_id, field_name):
            func = reduce(lambda obj, key: obj[key],
                          json.meta_metadata[field_name]['function'],
                          json.metadata.field_parser.field_definitions())
            return try_to_eval(func, json.metadata.functions, self=json)

        if args == cls.DEFAULT_TIMEOUT:
            return
        if action == 'get':
            if args == 0:  # No cached version is stored, retrieve it
                func = reduce(lambda obj, key: obj[key],
                              json.meta_metadata[field_name]['function'],
                              json.metadata.field_parser.field_definitions())
                json._dict_bson[field_name] = try_to_eval(
                    func, json.metadata.functions, self=json)
            else:
                json._dict_bson[field_name] = memoize(json.get('_id'),
                                                      field_name)
        elif action == 'set':
            if args >= 0:  # Don't store anything
                json._dict_bson[field_name] = None
def produce(self, fields=None):
    """Export the json in marc format.

    Produces a list of dictionaries will all the possible marc tags as keys.

    :param fields: list of fields to include in the output, if None or
                empty list all available tags will be included.
    """
    if not fields:
        fields = self.keys()

    out = []

    for field in fields:
        if field.startswith('__') or self.get(field) is None:
            continue
        json_id = self.meta_metadata[field]['json_id']
        values = self.get(field)
        if not isinstance(values, (list, tuple)):
            values = (values, )
        for value in values:
            try:
                for rule in get_producer_rules(
                        json_id, 'json_for_marc',
                        self.metadata):
                    marc_tags = rule[0] if isinstance(rule[0], tuple) \
                        else (rule[0], )
                    if marc_tags and not any(
                            [re.match(m, t)
                             for m in marc_tags
                             for t in self.meta_metadata[field]['function']]):
                        # Not match, continue to next rule
                        continue
                    tmp_dict = dict()
                    for marc_tag, subfield in iteritems(rule[1]):
                        if len(marc_tag) == 1:
                            marc_tag = \
                                self.meta_metadata[field]['function'][0] + \
                                marc_tag
                        if not subfield:
                            tmp_dict[marc_tag] = value
                        else:
                            try:
                                tmp_dict[marc_tag] = value[subfield]
                            except:
                                try:
                                    # Evaluate only non keyword values.
                                    if subfield in __builtins__:
                                        raise ImportError
                                    tmp_dict[marc_tag] = try_to_eval(
                                        subfield,
                                        self.metadata.functions,
                                        value=value,
                                        self=self)
                                except ImportError:
                                    pass
                                except Exception as e:
                                    self.continuable_errors.append(
                                        "Producer CError - Unable to produce "
                                        "'%s'.\n %s" % (field, str(e)))
                    if tmp_dict:
                        out.append(tmp_dict)
            except Exception as e:
                self.continuable_errors.append(
                    "Producer CError - Unable to produce '%s'.\n %s"
                    % (field, str(e)))
    return out
 def memoize(_id, field_name):
     func = reduce(lambda obj, key: obj[key],
                   json.meta_metadata[field_name]['function'],
                   json.metadata.field_parser.field_definitions())
     return try_to_eval(func, json.metadata.functions, self=json)
 def create_element(cls, rule, metadata):
     """Create the dictionary with the dump and load functions."""
     return {'loads': try_to_eval(rule.json_ext.loads,
                                  metadata.functions),
             'dumps': try_to_eval(rule.json_ext.dumps,
                                  metadata.functions)}
Beispiel #21
0
 def memoize(_id, field_name):
     func = reduce(lambda obj, key: obj[key],
                   json.meta_metadata[field_name]['function'],
                   json.metadata.field_parser.field_definitions())
     return try_to_eval(func, json.metadata.functions, self=json)
Beispiel #22
0
 def create_element(cls, rule, metadata):
     """Just evaluate the content of the schema to a python dictionary."""
     return try_to_eval(rule.schema, metadata.functions)
 def create_element(cls, rule, metadata):
     """Just evaluate the content of the schema to a python dictionary."""
     return try_to_eval(rule.schema, metadata.functions)
Beispiel #24
0
def produce(self, fields=None):
    """Export the json in marc format.

    Produces a list of dictionaries will all the possible marc tags as keys.

    :param fields: list of fields to include in the output, if None or
                empty list all available tags will be included.
    """
    if not fields:
        fields = self.keys()

    out = []

    for field in fields:
        if field.startswith('__') or self.get(field) is None:
            continue
        json_id = self.meta_metadata[field]['json_id']
        values = self.get(field)
        if not isinstance(values, (list, tuple)):
            values = (values, )
        for value in values:
            try:
                for rule in get_producer_rules(json_id, 'json_for_marc',
                                               self.metadata):
                    marc_tags = rule[0] if isinstance(rule[0], tuple) \
                        else (rule[0], )
                    if marc_tags and not any([
                            re.match(m, t) for m in marc_tags
                            for t in self.meta_metadata[field]['function']
                    ]):
                        # Not match, continue to next rule
                        continue
                    tmp_dict = dict()
                    for marc_tag, subfield in iteritems(rule[1]):
                        if len(marc_tag) == 1:
                            marc_tag = \
                                self.meta_metadata[field]['function'][0] + \
                                marc_tag
                        if not subfield:
                            tmp_dict[marc_tag] = value
                        else:
                            try:
                                tmp_dict[marc_tag] = value[subfield]
                            except:
                                try:
                                    # Evaluate only non keyword values.
                                    if subfield in __builtins__:
                                        raise ImportError
                                    tmp_dict[marc_tag] = try_to_eval(
                                        subfield,
                                        self.metadata.functions,
                                        value=value,
                                        self=self)
                                except ImportError:
                                    pass
                                except Exception as e:
                                    self.continuable_errors.append(
                                        "Producer CError - Unable to produce "
                                        "'%s'.\n %s" % (field, str(e)))
                    if tmp_dict:
                        out.append(tmp_dict)
            except Exception as e:
                self.continuable_errors.append(
                    "Producer CError - Unable to produce '%s'.\n %s" %
                    (field, str(e)))
    return out