def fill():
alldicts = {}
from invenio.bibrank_tag_based_indexer import fromDB
redis = get_redis()
serialized_weights = redis.get('citations_weights')
if serialized_weights:
weights = deserialize_via_marshal(serialized_weights)
else:
weights = fromDB('citation')
alldicts['citations_weights'] = weights
# for cited:M->N queries, it is interesting to cache also
# some preprocessed citationdict:
alldicts['citations_keys'] = intbitset(weights.keys())
# Citation counts
alldicts['citations_counts'] = [t for t in weights.iteritems()]
alldicts['citations_counts'].sort(key=itemgetter(1), reverse=True)
# Self-cites
serialized_weights = redis.get('selfcites_weights')
if serialized_weights:
selfcites = deserialize_via_marshal(serialized_weights)
else:
selfcites = fromDB('selfcites')
selfcites_weights = {}
for recid, counts in alldicts['citations_counts']:
selfcites_weights[recid] = counts - selfcites.get(recid, 0)
alldicts['selfcites_weights'] = selfcites_weights
alldicts['selfcites_counts'] = [(recid, selfcites_weights.get(recid, cites)) for recid, cites in alldicts['citations_counts']]
alldicts['selfcites_counts'].sort(key=itemgetter(1), reverse=True)
return alldicts
def bst_webcoll_postprocess(recids=[]):
"""Parse recids to POST to remote server to alert that records are visible."""
if isinstance(recids, str):
recids = recids.split(",")
cache = get_redis()
cached_ids = cache.get("webcoll_pending_recids") or []
if cached_ids and not cached_ids == "[]":
if isinstance(cached_ids, str):
cached_ids = eval(cached_ids)
recids += cached_ids
if not CFG_WEBCOLL_POST_REQUEST_URL:
write_message("CFG_WEBCOLL_POST_REQUEST_URL is not set.")
return
if recids and len(recids) > 0 and not recids == "[]":
write_message("Going to POST callback to {0}: {1} (total: {2})".format(
CFG_WEBCOLL_POST_REQUEST_URL,
recids[:10],
len(recids))
)
cache.set("webcoll_pending_recids", recids)
session = requests.Session()
try:
addapter = requests.adapters.HTTPAdapter(max_retries=3)
session.mount(CFG_WEBCOLL_POST_REQUEST_URL, addapter)
response = session.post(CFG_WEBCOLL_POST_REQUEST_URL,
data={'recids': recids})
except Exception as err:
write_message("Post request failed!")
write_message(err)
return
finally:
session.close()
if response.ok:
write_message("Post request sent successfully")
cache.set("webcoll_pending_recids", [])
else:
write_message("Post request failed!")
write_message(response.text)
cache.set("webcoll_pending_recids", recids)
else:
write_message("No recids to POST callback for to {0}.".format(
CFG_WEBCOLL_POST_REQUEST_URL,
))
def bst_webcoll_postprocess(recids=[]):
"""Parse recids to POST to remote server to alert that records are visible."""
if isinstance(recids, str):
recids = recids.split(",")
cache = get_redis()
cached_ids = cache.get("webcoll_pending_recids") or []
if cached_ids and not cached_ids == "[]":
if isinstance(cached_ids, str):
cached_ids = eval(cached_ids)
recids += cached_ids
if not CFG_WEBCOLL_POST_REQUEST_URL:
write_message("CFG_WEBCOLL_POST_REQUEST_URL is not set.")
return
if recids and len(recids) > 0 and not recids == "[]":
write_message("Going to POST callback to {0}: {1} (total: {2})".format(
CFG_WEBCOLL_POST_REQUEST_URL, recids[:10], len(recids)))
cache.set("webcoll_pending_recids", recids)
session = requests.Session()
try:
addapter = requests.adapters.HTTPAdapter(max_retries=3)
session.mount(CFG_WEBCOLL_POST_REQUEST_URL, addapter)
response = session.post(CFG_WEBCOLL_POST_REQUEST_URL,
data={'recids': recids})
except Exception as err:
write_message("Post request failed!")
write_message(err)
return
finally:
session.close()
if response.ok:
write_message("Post request sent successfully")
cache.set("webcoll_pending_recids", [])
else:
write_message("Post request failed!")
write_message(response.text)
cache.set("webcoll_pending_recids", recids)
else:
write_message("No recids to POST callback for to {0}.".format(
CFG_WEBCOLL_POST_REQUEST_URL, ))
def bst_webcoll_postprocess(recids=[]):
"""Parse recids to POST to remote server to alert that records are visible."""
if isinstance(recids, str):
recids = recids.split(",")
cache = get_redis()
cached_ids = cache.get("webcoll_pending_recids") or []
recids += cached_ids
if not CFG_WEBCOLL_POST_REQUEST_URL:
write_message("CFG_WEBCOLL_POST_REQUEST_URL is not set.")
task_update_status('ERROR')
return 1
if recids:
write_message("Going to POST callback to {0}: {1} (total: {2})".format(
CFG_WEBCOLL_POST_REQUEST_URL,
recids[:10],
len(recids))
)
session = requests.Session()
addapter = requests.adapters.HTTPAdapter(max_retries=3)
session.mount(CFG_WEBCOLL_POST_REQUEST_URL, addapter)
response = session.post(CFG_WEBCOLL_POST_REQUEST_URL,
data={'recids': recids})
if response.ok:
write_message("Post request sent successfully")
cache.set("webcoll_pending_recids", [])
else:
write_message("Post request failed!")
write_message(response.text)
task_update_status('ERROR')
cache.set("webcoll_pending_recids", recids)
session.close()
else:
write_message("No recids to POST callback for to {0}.".format(
CFG_WEBCOLL_POST_REQUEST_URL,
))
def save_in_storage(self, sid, session_object, timeout, uid): # pylint: disable=W0613
return get_redis().setex(self.generate_key(sid), session_object, timeout)
def delete_from_storage(self, sid):
return get_redis().delete(self.generate_key(sid))
def load_from_storage(self, sid):
return get_redis().get(self.generate_key(sid))
def store_weights_cache(weights):
"""Store into key/value store"""
redis = get_redis()
redis.set('selfcites_weights', serialize_via_marshal(weights))
def test_expire(self):
db = get_redis()
db.set('hello', 'a', 1)
time.sleep(2)
self.assertEqual(db.get('hello'), None)
def test_dummy(self):
db = get_redis()
db.set('hello', 'a')
self.assertEqual(db.get('hello'), None)
db.delete('hello')
def test_simple(self):
db = get_redis()
db.set('hello_test', 'a')
self.assertEqual(db.get('hello_test'), 'a')
db.delete('hello_test')
def store_weights_cache(weights):
"""Store into key/value store"""
redis = get_redis()
redis.set('selfcites_weights', serialize_via_marshal(weights))
def save_in_storage(self, sid, session_object, timeout, uid): # pylint: disable=W0613
return get_redis().setex(self.generate_key(sid), session_object,
timeout)
def delete_from_storage(self, sid):
return get_redis().delete(self.generate_key(sid))
def load_from_storage(self, sid):
return get_redis().get(self.generate_key(sid))
