Esempio n. 1
0
    def __init__(self):
        def cache_filler():
            alldicts = {}
            try:
                res = run_sql("SELECT object_name,object_value FROM rnkCITATIONDATA")
            except OperationalError:
                # database problems, return empty cache
                return {}
            for row in res:
                object_name = row[0]
                object_value = row[1]
                try:
                    object_value_dict = deserialize_via_marshal(object_value)
                except:
                    object_value_dict = {}
                alldicts[object_name] = object_value_dict
                if object_name == 'citationdict':
                    # for cited:M->N queries, it is interesting to cache also
                    # some preprocessed citationdict:
                    alldicts['citationdict_keys'] = object_value_dict.keys()
                    alldicts['citationdict_keys_intbitset'] = intbitset(object_value_dict.keys())
            return alldicts
        def timestamp_verifier():
            res = run_sql("""SELECT DATE_FORMAT(last_updated, '%Y-%m-%d %H:%i:%s')
                             FROM rnkMETHOD WHERE name='citation'""")
            if res:
                return res[0][0]
            else:
                return '0000-00-00 00:00:00'

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 2
0
    def __init__(self):
        def cache_filler():
            return dict(
                IdxINDEX.query.values(IdxINDEX.id, IdxINDEX.stemming_language))

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return get_table_update_time('idxINDEX')

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 3
0
    def __init__(self):
        def cache_filler():
            return dict(IdxINDEX.query.values(IdxINDEX.id,
                                              IdxINDEX.stemming_language))

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return get_table_update_time('idxINDEX')

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 4
0
    def __init__(self):
        def fill():
            alldicts = {}
            from invenio.legacy.bibrank.tag_based_indexer import fromDB
            redis = get_redis()
            serialized_weights = redis.get('citations_weights')
            if serialized_weights:
                weights = deserialize_via_marshal(serialized_weights)
            else:
                weights = fromDB('citation')

            alldicts['citations_weights'] = weights
            # for cited:M->N queries, it is interesting to cache also
            # some preprocessed citationdict:
            alldicts['citations_keys'] = intbitset(weights.keys())

            # Citation counts
            alldicts['citations_counts'] = [t for t in iteritems(weights)]
            alldicts['citations_counts'].sort(key=itemgetter(1), reverse=True)

            # Self-cites
            serialized_weights = redis.get('selfcites_weights')
            if serialized_weights:
                selfcites = deserialize_via_marshal(serialized_weights)
            else:
                selfcites = fromDB('selfcites')
            selfcites_weights = {}
            for recid, counts in alldicts['citations_counts']:
                selfcites_weights[recid] = counts - selfcites.get(recid, 0)
            alldicts['selfcites_weights'] = selfcites_weights
            alldicts['selfcites_counts'] = [
                (recid, selfcites_weights.get(recid, cites))
                for recid, cites in alldicts['citations_counts']
            ]
            alldicts['selfcites_counts'].sort(key=itemgetter(1), reverse=True)

            return alldicts

        def cache_filler():
            self.cache = None  # misfire from pylint: disable=W0201
            # this is really defined in DataCacher
            return fill()

        from invenio.legacy.bibrank.tag_based_indexer import get_lastupdated

        def timestamp_verifier():
            citation_lastupdate = get_lastupdated('citation')
            if citation_lastupdate:
                return citation_lastupdate.strftime("%Y-%m-%d %H:%M:%S")
            else:
                return "0000-00-00 00:00:00"

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 5
0
    def __init__(self):
        def fill():
            alldicts = {}
            from invenio.legacy.bibrank.tag_based_indexer import fromDB

            serialized_weights = cache.get("citations_weights")
            if serialized_weights:
                weights = deserialize_via_marshal(serialized_weights)
            else:
                weights = fromDB("citation")

            alldicts["citations_weights"] = weights
            # for cited:M->N queries, it is interesting to cache also
            # some preprocessed citationdict:
            alldicts["citations_keys"] = intbitset(weights.keys())

            # Citation counts
            alldicts["citations_counts"] = [t for t in iteritems(weights)]
            alldicts["citations_counts"].sort(key=itemgetter(1), reverse=True)

            # Self-cites
            serialized_weights = cache.get("selfcites_weights")
            if serialized_weights:
                selfcites = deserialize_via_marshal(serialized_weights)
            else:
                selfcites = fromDB("selfcites")
            selfcites_weights = {}
            for recid, counts in alldicts["citations_counts"]:
                selfcites_weights[recid] = counts - selfcites.get(recid, 0)
            alldicts["selfcites_weights"] = selfcites_weights
            alldicts["selfcites_counts"] = [
                (recid, selfcites_weights.get(recid, cites)) for recid, cites in alldicts["citations_counts"]
            ]
            alldicts["selfcites_counts"].sort(key=itemgetter(1), reverse=True)

            return alldicts

        def cache_filler():
            self.cache = None  # misfire from pylint: disable=W0201
            # this is really defined in DataCacher
            return fill()

        from invenio.legacy.bibrank.tag_based_indexer import get_lastupdated

        def timestamp_verifier():
            citation_lastupdate = get_lastupdated("citation")
            if citation_lastupdate:
                return citation_lastupdate.strftime("%Y-%m-%d %H:%M:%S")
            else:
                return "0000-00-00 00:00:00"

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 6
0
    def __init__(self, method_name):
        """Initialize data cacher for given method."""
        self.method_name = method_name

        def cache_filler():
            """Return data to populate cache."""
            method = BsrMETHOD.query.filter_by(name=self.method_name).first()
            return dict(method.get_cache()) if method is not None else {}

        def timestamp_verifier():
            """Return string representing last update datetime."""
            return BsrMETHOD.timestamp_verifier(self.method_name).strftime(
                "%Y-%m-%d %H:%M:%S")

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 7
0
    def __init__(self):
        """Initilize cache."""
        def cache_filler():
            collections = Collection.query.all()
            collection_index = dict([(c.id, c.name) for c in collections])

            return dict([(c.name, map(collection_index.get, c.descendants_ids))
                         for c in collections])

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return max(get_table_update_time('collection'),
                       get_table_update_time('collection_collection'))

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 8
0
    def __init__(self, method_name):
        """Initialize data cacher for given method."""
        self.method_name = method_name

        def cache_filler():
            """Return data to populate cache."""
            method = BsrMETHOD.query.filter_by(name=self.method_name).first()
            return dict(method.get_cache()) if method is not None else {}

        def timestamp_verifier():
            """Return string representing last update datetime."""
            return BsrMETHOD.timestamp_verifier(
                self.method_name).strftime("%Y-%m-%d %H:%M:%S")

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 9
0
    def __init__(self):
        def cache_filler():
            from invenio.modules.search.searchext.engines.native import \
                search_unit_in_idxphrases
            collections = Collection.query.all()
            setattr(get_all_recids, 'cache', dict())
            setattr(get_collection_nbrecs, 'cache', dict())
            return dict([(c.name,
                          search_unit_in_idxphrases(c.name, 'collection', 'e'))
                         for c in collections])

        def timestamp_verifier():
            return IdxINDEX.query.filter_by(id=self._index_id).value(
                'last_updated').strftime("%Y-%m-%d %H:%M:%S")

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 10
0
    def __init__(self):
        def cache_filler():
            from invenio.modules.search.searchext.engines.native import \
                search_unit_in_idxphrases
            collections = Collection.query.all()
            setattr(get_all_recids, 'cache', dict())
            setattr(get_collection_nbrecs, 'cache', dict())
            return dict([(c.name,
                          search_unit_in_idxphrases(c.name, 'collection', 'e'))
                         for c in collections])

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return get_table_update_time('collection')

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 11
0
    def __init__(self):
        def cache_filler():
            from invenio.modules.search.searchext.engines.native import \
                search_unit_in_idxphrases
            collections = Collection.query.all()
            setattr(get_all_recids, 'cache', dict())
            setattr(get_collection_nbrecs, 'cache', dict())
            return dict([
                (c.name, search_unit_in_idxphrases(c.name, 'collection', 'e'))
                for c in collections
            ])

        def timestamp_verifier():
            return IdxINDEX.query.filter_by(id=self._index_id).value(
                'last_updated').strftime("%Y-%m-%d %H:%M:%S")

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 12
0
    def __init__(self):
        def cache_filler():
            res = Field.query.join(Fieldname).filter(
                Fieldname.type == 'ln').values(Field.name, 'ln', 'value')
            ret = {}
            for f, ln, i18nname in res:
                if i18nname:
                    if f not in ret:
                        ret[f] = {}
                    ret[f][ln] = i18nname
            return ret

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return get_table_update_time('fieldname')

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 13
0
    def __init__(self):
        def cache_filler():
            from invenio.modules.search.searchext.engines.native import \
                search_unit_in_idxphrases
            collections = Collection.query.all()
            setattr(get_all_recids, 'cache', dict())
            setattr(get_collection_nbrecs, 'cache', dict())
            return dict([
                (c.name, search_unit_in_idxphrases(c.name, 'collection', 'e'))
                for c in collections
            ])

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return get_table_update_time('collection')

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 14
0
    def __init__(self):
        """Initilize cache."""
        def cache_filler():
            collections = Collection.query.all()
            collection_index = dict([(c.id, c.name) for c in collections])

            return dict([
                (c.name, map(collection_index.get, c.descendants_ids))
                for c in collections
            ])

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return max(get_table_update_time('collection'),
                       get_table_update_time('collection_collection'))

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 15
0
    def __init__(self):
        def cache_filler():
            res = Field.query.join(Fieldname).filter(
                Fieldname.type == 'ln'
            ).values(Field.name, 'ln', 'value')
            ret = {}
            for f, ln, i18nname in res:
                if i18nname:
                    if f not in ret:
                        ret[f] = {}
                    ret[f][ln] = i18nname
            return ret

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return get_table_update_time('fieldname')

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 16
0
    def __init__(self):
        def cache_filler():
            res = Collection.query.join(Collection.collection_names).filter(
                Collectionname.type == 'ln').values(Collection.name, 'ln',
                                                    'value')
            ret = {}
            for c, ln, i18nname in res:
                if i18nname:
                    if c not in ret:
                        ret[c] = {}
                    ret[c][ln] = i18nname
            return ret

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return get_table_update_time('collectionname')

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 17
0
    def __init__(self):
        def cache_filler():
            res = Collection.query.join(
                Collection.collection_names
            ).filter(Collectionname.type == 'ln').values(
                Collection.name, 'ln', 'value'
            )
            ret = {}
            for c, ln, i18nname in res:
                if i18nname:
                    if c not in ret:
                        ret[c] = {}
                    ret[c][ln] = i18nname
            return ret

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return get_table_update_time('collectionname')

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 18
0
    def __init__(self):
        def cache_filler():
            from invenio.modules.access.control import acc_get_action_id
            from invenio.modules.access.local_config import VIEWRESTRCOLL
            from invenio.modules.access.models import (AccAuthorization,
                                                       AccARGUMENT)
            VIEWRESTRCOLL_ID = acc_get_action_id(VIEWRESTRCOLL)

            return [
                auth[0] for auth in
                AccAuthorization.query.join(AccAuthorization.argument).filter(
                    AccARGUMENT.keyword == 'collection', AccAuthorization.
                    id_accACTION == VIEWRESTRCOLL_ID).values(AccARGUMENT.value)
            ]

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return max(get_table_update_time('accROLE_accACTION_accARGUMENT'),
                       get_table_update_time('accARGUMENT'))

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 19
0
    def __init__(self):
        def cache_filler():
            from invenio.modules.access.control import acc_get_action_id
            from invenio.modules.access.local_config import VIEWRESTRCOLL
            from invenio.modules.access.models import (
                AccAuthorization, AccARGUMENT
            )
            VIEWRESTRCOLL_ID = acc_get_action_id(VIEWRESTRCOLL)

            return [auth[0] for auth in AccAuthorization.query.join(
                AccAuthorization.argument
            ).filter(
                AccARGUMENT.keyword == 'collection',
                AccAuthorization.id_accACTION == VIEWRESTRCOLL_ID
            ).values(AccARGUMENT.value)]

        def timestamp_verifier():
            from invenio.legacy.dbquery import get_table_update_time
            return max(get_table_update_time('accROLE_accACTION_accARGUMENT'),
                       get_table_update_time('accARGUMENT'))

        DataCacher.__init__(self, cache_filler, timestamp_verifier)
Esempio n. 20
0
    def get_data_cache(self, recreate_cache_if_needed=True):
        """Return always up-to-date data from cache.

        The returned value depends on the what has been stored with
        :meth:`~SearchService.prepare_data_cache`.

        :param recreate_cache_if_needed: if True, force refreshing data cache.
        """
        try:
            self.__class__.cache.is_ok_p
        except Exception:
            self.__class__.cache = DataCacher(self.prepare_data_cache,
                                              self.timestamp_verifier)
        if recreate_cache_if_needed:
            self.__class__.cache.recreate_cache_if_needed()

        return self.__class__.cache.cache