def test_put_and_get_no_namespace(self):
cache = RedisCache()
cache.put("unittestkey1", "val1")
actual = self._redis.get("unittestkey1")
expected = "val1"
self.assertEqual(expected, actual)
actual = cache.get("unittestkey1")
expected = "val1"
self.assertEqual(expected, actual)
def test_setting_cachesecs(self):
cache = RedisCache(namespace="unittest:", cachesecs=2)
cache.put("key1", "val1")
time.sleep(1)
actual = cache.get("key1")
expected = "val1"
self.assertEqual(expected, actual)
time.sleep(1.5)
actual = cache.get("key1")
expected = None
self.assertEqual(expected, actual)
def test_put_and_get_with_namespace(self):
cache = RedisCache(namespace="unittest:")
cache.put("key1", "val1")
actual = self._redis.get("unittest:key1")
expected = "val1"
self.assertEqual(expected, actual)
actual = self._redis.get("key1")
expected = None
self.assertEqual(expected, actual)
actual = cache.get("key1")
expected = "val1"
self.assertEqual(expected, actual)
def test_delete(self):
cache = RedisCache(namespace="unittest:")
cache.put("key1", "val1")
actual = cache.get("key1")
expected = "val1"
self.assertEqual(expected, actual)
cache.delete("key1")
actual = cache.get("key1")
expected = None
self.assertEqual(expected, actual)
def __init__(self, storeid):
"""Args:
storeid - ID from settings.py MongoStores config.
"""
self._store = None
self._storeid = storeid
try:
storeconf = MongoStores[storeid]
except KeyError:
raise Exception("Unrecognized store ID: %s" % storeid)
cachetype = storeconf["cachetype"]
cache_namespace = storeconf.get("cache_namespace")
cachesecs = storeconf.get("cachesecs")
# create the cache
if cachetype == CacheType.Off:
self._cache = None
elif cachetype == CacheType.Volatile:
self._cache = Memcache(namespace=cache_namespace)
elif cachetype == CacheType.Persistent:
self._cache = RedisCache(cachesecs=cachesecs, namespace=cache_namespace)
else:
raise Exception("Unknown cache type enumeration %s" % cachetype)
def cache_top_tweets():
#initialize stores
ts = int(time.time())
countstore = CountStore()
tweetstore = TweetStore()
cache = RedisCache(namespace=settings.TopTweetsCache["namespace"])
countries = Geo.country_codes()
top_tweets_cache = {}
for country in countries:
print "*************"
print country
print "*************"
top_tweets = {}
segmentation = "C" + country
for entitytype in ["hashtag", "user_mention"]:
top_tweets[entitytype] = []
top_entities = countstore.get_top(entitytype,
segmentation,
settings.Aggregation['top_entities'],
ts)
for entity, count in top_entities:
data = {"text":entity, "count":count, "tweets":[]}
tweets = top_tweets_cache.get((entitytype, entity, ts))
if not tweets:
print "fetching tweets for " + str((entitytype, entity, ts))
segmentation = ":".join([entitytype, entity])
tweets = countstore.get_top("tweet",
segmentation,
settings.Aggregation['top_tweets'],
ts)
tweets = map(lambda x: (tweetstore.get(x[0]), x[1]), tweets)
top_tweets_cache[(entitytype, entity, ts)] = tweets
for tweet, count in tweets:
data["tweets"].append({"tweet":tweet.data, "count": count})
top_tweets[entitytype].append(data)
cache.put(segmentation, top_tweets)
def test_expire(self):
cache = RedisCache(namespace="unittest:")
cache.put("key1", "val1")
actual = cache.get("key1")
expected = "val1"
self.assertEqual(expected, actual)
cache.expire("key1", 2)
time.sleep(1)
actual = cache.get("key1")
expected = "val1"
self.assertEqual(expected, actual)
time.sleep(1.5)
actual = cache.get("key1")
expected = None
self.assertEqual(expected, actual)
class MongoStore(object):
"""Wrapper around mongo access that handles caching."""
def __init__(self, storeid):
"""Args:
storeid - ID from settings.py MongoStores config.
"""
self._store = None
self._storeid = storeid
try:
storeconf = MongoStores[storeid]
except KeyError:
raise Exception("Unrecognized store ID: %s" % storeid)
cachetype = storeconf["cachetype"]
cache_namespace = storeconf.get("cache_namespace")
cachesecs = storeconf.get("cachesecs")
# create the cache
if cachetype == CacheType.Off:
self._cache = None
elif cachetype == CacheType.Volatile:
self._cache = Memcache(namespace=cache_namespace)
elif cachetype == CacheType.Persistent:
self._cache = RedisCache(cachesecs=cachesecs, namespace=cache_namespace)
else:
raise Exception("Unknown cache type enumeration %s" % cachetype)
@property
def store(self):
if not self._store:
self._store = MongoFactory.getconn(self._storeid)
return self._store
def _get_cached(self, key):
if self._cache:
return self._cache.get(key)
return None
def _set_cache(self, key, val):
if self._cache:
self._cache.put(key, val)
def find_one(self, key):
result = self._get_cached(key)
if result:
return result
result = self.store.find_one(key)
return result
def save(self, doc):
key = self.store.save(doc)
self._set_cache(key, doc)
return key
def find(self, query=None, fields=None):
query = query or {}
if fields:
cursor = self.store.find(query, fields)
else:
cursor = self.store.find(query)
return cursor
def count(self):
return self.store.count()
def update(self, query, update_dict, upsert=False):
self.store.update(query, update_dict, upsert=upsert)
if self._cache:
new_doc = self.store.find_one(query)
key = new_doc["_id"]
self._set_cache(key, new_doc)
def delete_one(self, key):
self.store.remove({"_id": key})
if self._cache:
self._cache.delete(key)
def test_non_string_storage(self):
cache = RedisCache(namespace="unittest:")
cache.put("key1", {"hello": "world"})
actual = cache.get("key1")
expected = {"hello": "world"}
self.assertEqual(expected, actual)