def __init__(self):
def cache_filler():
alldicts = {}
try:
res = run_sql("SELECT object_name,object_value FROM rnkCITATIONDATA")
except OperationalError:
# database problems, return empty cache
return {}
for row in res:
object_name = row[0]
object_value = row[1]
try:
object_value_dict = deserialize_via_marshal(object_value)
except:
object_value_dict = {}
alldicts[object_name] = object_value_dict
if object_name == 'citationdict':
# for cited:M->N queries, it is interesting to cache also
# some preprocessed citationdict:
alldicts['citationdict_keys'] = object_value_dict.keys()
alldicts['citationdict_keys_intbitset'] = intbitset(object_value_dict.keys())
return alldicts
def timestamp_verifier():
res = run_sql("""SELECT DATE_FORMAT(last_updated, '%Y-%m-%d %H:%i:%s')
FROM rnkMETHOD WHERE name='citation'""")
if res:
return res[0][0]
else:
return '0000-00-00 00:00:00'
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
return dict(
IdxINDEX.query.values(IdxINDEX.id, IdxINDEX.stemming_language))
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return get_table_update_time('idxINDEX')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
return dict(IdxINDEX.query.values(IdxINDEX.id,
IdxINDEX.stemming_language))
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return get_table_update_time('idxINDEX')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def fill():
alldicts = {}
from invenio.legacy.bibrank.tag_based_indexer import fromDB
redis = get_redis()
serialized_weights = redis.get('citations_weights')
if serialized_weights:
weights = deserialize_via_marshal(serialized_weights)
else:
weights = fromDB('citation')
alldicts['citations_weights'] = weights
# for cited:M->N queries, it is interesting to cache also
# some preprocessed citationdict:
alldicts['citations_keys'] = intbitset(weights.keys())
# Citation counts
alldicts['citations_counts'] = [t for t in iteritems(weights)]
alldicts['citations_counts'].sort(key=itemgetter(1), reverse=True)
# Self-cites
serialized_weights = redis.get('selfcites_weights')
if serialized_weights:
selfcites = deserialize_via_marshal(serialized_weights)
else:
selfcites = fromDB('selfcites')
selfcites_weights = {}
for recid, counts in alldicts['citations_counts']:
selfcites_weights[recid] = counts - selfcites.get(recid, 0)
alldicts['selfcites_weights'] = selfcites_weights
alldicts['selfcites_counts'] = [
(recid, selfcites_weights.get(recid, cites))
for recid, cites in alldicts['citations_counts']
]
alldicts['selfcites_counts'].sort(key=itemgetter(1), reverse=True)
return alldicts
def cache_filler():
self.cache = None # misfire from pylint: disable=W0201
# this is really defined in DataCacher
return fill()
from invenio.legacy.bibrank.tag_based_indexer import get_lastupdated
def timestamp_verifier():
citation_lastupdate = get_lastupdated('citation')
if citation_lastupdate:
return citation_lastupdate.strftime("%Y-%m-%d %H:%M:%S")
else:
return "0000-00-00 00:00:00"
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def fill():
alldicts = {}
from invenio.legacy.bibrank.tag_based_indexer import fromDB
serialized_weights = cache.get("citations_weights")
if serialized_weights:
weights = deserialize_via_marshal(serialized_weights)
else:
weights = fromDB("citation")
alldicts["citations_weights"] = weights
# for cited:M->N queries, it is interesting to cache also
# some preprocessed citationdict:
alldicts["citations_keys"] = intbitset(weights.keys())
# Citation counts
alldicts["citations_counts"] = [t for t in iteritems(weights)]
alldicts["citations_counts"].sort(key=itemgetter(1), reverse=True)
# Self-cites
serialized_weights = cache.get("selfcites_weights")
if serialized_weights:
selfcites = deserialize_via_marshal(serialized_weights)
else:
selfcites = fromDB("selfcites")
selfcites_weights = {}
for recid, counts in alldicts["citations_counts"]:
selfcites_weights[recid] = counts - selfcites.get(recid, 0)
alldicts["selfcites_weights"] = selfcites_weights
alldicts["selfcites_counts"] = [
(recid, selfcites_weights.get(recid, cites)) for recid, cites in alldicts["citations_counts"]
]
alldicts["selfcites_counts"].sort(key=itemgetter(1), reverse=True)
return alldicts
def cache_filler():
self.cache = None # misfire from pylint: disable=W0201
# this is really defined in DataCacher
return fill()
from invenio.legacy.bibrank.tag_based_indexer import get_lastupdated
def timestamp_verifier():
citation_lastupdate = get_lastupdated("citation")
if citation_lastupdate:
return citation_lastupdate.strftime("%Y-%m-%d %H:%M:%S")
else:
return "0000-00-00 00:00:00"
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self, method_name):
"""Initialize data cacher for given method."""
self.method_name = method_name
def cache_filler():
"""Return data to populate cache."""
method = BsrMETHOD.query.filter_by(name=self.method_name).first()
return dict(method.get_cache()) if method is not None else {}
def timestamp_verifier():
"""Return string representing last update datetime."""
return BsrMETHOD.timestamp_verifier(self.method_name).strftime(
"%Y-%m-%d %H:%M:%S")
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
"""Initilize cache."""
def cache_filler():
collections = Collection.query.all()
collection_index = dict([(c.id, c.name) for c in collections])
return dict([(c.name, map(collection_index.get, c.descendants_ids))
for c in collections])
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return max(get_table_update_time('collection'),
get_table_update_time('collection_collection'))
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self, method_name):
"""Initialize data cacher for given method."""
self.method_name = method_name
def cache_filler():
"""Return data to populate cache."""
method = BsrMETHOD.query.filter_by(name=self.method_name).first()
return dict(method.get_cache()) if method is not None else {}
def timestamp_verifier():
"""Return string representing last update datetime."""
return BsrMETHOD.timestamp_verifier(
self.method_name).strftime("%Y-%m-%d %H:%M:%S")
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
from invenio.modules.search.searchext.engines.native import \
search_unit_in_idxphrases
collections = Collection.query.all()
setattr(get_all_recids, 'cache', dict())
setattr(get_collection_nbrecs, 'cache', dict())
return dict([(c.name,
search_unit_in_idxphrases(c.name, 'collection', 'e'))
for c in collections])
def timestamp_verifier():
return IdxINDEX.query.filter_by(id=self._index_id).value(
'last_updated').strftime("%Y-%m-%d %H:%M:%S")
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
from invenio.modules.search.searchext.engines.native import \
search_unit_in_idxphrases
collections = Collection.query.all()
setattr(get_all_recids, 'cache', dict())
setattr(get_collection_nbrecs, 'cache', dict())
return dict([(c.name,
search_unit_in_idxphrases(c.name, 'collection', 'e'))
for c in collections])
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return get_table_update_time('collection')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
from invenio.modules.search.searchext.engines.native import \
search_unit_in_idxphrases
collections = Collection.query.all()
setattr(get_all_recids, 'cache', dict())
setattr(get_collection_nbrecs, 'cache', dict())
return dict([
(c.name, search_unit_in_idxphrases(c.name, 'collection', 'e'))
for c in collections
])
def timestamp_verifier():
return IdxINDEX.query.filter_by(id=self._index_id).value(
'last_updated').strftime("%Y-%m-%d %H:%M:%S")
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
res = Field.query.join(Fieldname).filter(
Fieldname.type == 'ln').values(Field.name, 'ln', 'value')
ret = {}
for f, ln, i18nname in res:
if i18nname:
if f not in ret:
ret[f] = {}
ret[f][ln] = i18nname
return ret
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return get_table_update_time('fieldname')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
from invenio.modules.search.searchext.engines.native import \
search_unit_in_idxphrases
collections = Collection.query.all()
setattr(get_all_recids, 'cache', dict())
setattr(get_collection_nbrecs, 'cache', dict())
return dict([
(c.name, search_unit_in_idxphrases(c.name, 'collection', 'e'))
for c in collections
])
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return get_table_update_time('collection')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
"""Initilize cache."""
def cache_filler():
collections = Collection.query.all()
collection_index = dict([(c.id, c.name) for c in collections])
return dict([
(c.name, map(collection_index.get, c.descendants_ids))
for c in collections
])
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return max(get_table_update_time('collection'),
get_table_update_time('collection_collection'))
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
res = Field.query.join(Fieldname).filter(
Fieldname.type == 'ln'
).values(Field.name, 'ln', 'value')
ret = {}
for f, ln, i18nname in res:
if i18nname:
if f not in ret:
ret[f] = {}
ret[f][ln] = i18nname
return ret
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return get_table_update_time('fieldname')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
res = Collection.query.join(Collection.collection_names).filter(
Collectionname.type == 'ln').values(Collection.name, 'ln',
'value')
ret = {}
for c, ln, i18nname in res:
if i18nname:
if c not in ret:
ret[c] = {}
ret[c][ln] = i18nname
return ret
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return get_table_update_time('collectionname')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
res = Collection.query.join(
Collection.collection_names
).filter(Collectionname.type == 'ln').values(
Collection.name, 'ln', 'value'
)
ret = {}
for c, ln, i18nname in res:
if i18nname:
if c not in ret:
ret[c] = {}
ret[c][ln] = i18nname
return ret
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return get_table_update_time('collectionname')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
from invenio.modules.access.control import acc_get_action_id
from invenio.modules.access.local_config import VIEWRESTRCOLL
from invenio.modules.access.models import (AccAuthorization,
AccARGUMENT)
VIEWRESTRCOLL_ID = acc_get_action_id(VIEWRESTRCOLL)
return [
auth[0] for auth in
AccAuthorization.query.join(AccAuthorization.argument).filter(
AccARGUMENT.keyword == 'collection', AccAuthorization.
id_accACTION == VIEWRESTRCOLL_ID).values(AccARGUMENT.value)
]
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return max(get_table_update_time('accROLE_accACTION_accARGUMENT'),
get_table_update_time('accARGUMENT'))
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def __init__(self):
def cache_filler():
from invenio.modules.access.control import acc_get_action_id
from invenio.modules.access.local_config import VIEWRESTRCOLL
from invenio.modules.access.models import (
AccAuthorization, AccARGUMENT
)
VIEWRESTRCOLL_ID = acc_get_action_id(VIEWRESTRCOLL)
return [auth[0] for auth in AccAuthorization.query.join(
AccAuthorization.argument
).filter(
AccARGUMENT.keyword == 'collection',
AccAuthorization.id_accACTION == VIEWRESTRCOLL_ID
).values(AccARGUMENT.value)]
def timestamp_verifier():
from invenio.legacy.dbquery import get_table_update_time
return max(get_table_update_time('accROLE_accACTION_accARGUMENT'),
get_table_update_time('accARGUMENT'))
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def get_data_cache(self, recreate_cache_if_needed=True):
"""Return always up-to-date data from cache.
The returned value depends on the what has been stored with
:meth:`~SearchService.prepare_data_cache`.
:param recreate_cache_if_needed: if True, force refreshing data cache.
"""
try:
self.__class__.cache.is_ok_p
except Exception:
self.__class__.cache = DataCacher(self.prepare_data_cache,
self.timestamp_verifier)
if recreate_cache_if_needed:
self.__class__.cache.recreate_cache_if_needed()
return self.__class__.cache.cache