def disable_caching(self): self.cache = CacheDisabled()
def __init__(self, name, cache_timeout=None): "When cache_only is true, the data will be stored in memcached only (used in tests)" self.name = name self.lpm = LargePersistentMap(name) self.cache = Cache(cache_timeout)
class LargePersistentCachedMap(object): """ A key:value dictionary which is saved both in Memcached and AWS DynamoDB. Create a map by simply calling the constructor with the map name. Supported value types are strings and numbers. Maximum value size is 60kb. m = LargePersistentCachedMap("my_table") m["key1"] = "some string" m["key2"] = set(["d", "e"])31415 m2 = LargePersistentCachedMap("my_table") print m2["key1"] """ def __init__(self, name, cache_timeout=None): "When cache_only is true, the data will be stored in memcached only (used in tests)" self.name = name self.lpm = LargePersistentMap(name) self.cache = Cache(cache_timeout) def __setitem__(self, key, value): Signals.pre_update.send(sender=self.__class__, map_name=self.name, key=key, action='put') cmp_key = LPCMKey(self.name, key) self.cache.set(cmp_key.cache_key, value) self.lpm[key] = value Signals.post_update.send(sender=self.__class__, map_name=self.name, key=key, action='put') def __getitem__(self, key): cmp_key = LPCMKey(self.name, key) v = self.cache.get(cmp_key.cache_key) if v is not None: return v return self._get_from_dynamodb_and_save_in_cache(cmp_key) def get(self, k, d=None): """ D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None. """ try: return self[k] except KeyError: return d def __contains__(self, key): try: v = self[key] except KeyError: return False return True def disable_caching(self): self.cache = CacheDisabled() def _get_from_dynamodb_and_save_in_cache(self, cmp_key): """ Gets value from dynamodb and puts it in cache, taking care of the cache-miss stampede """ if self.cache.get_thread_safe_token(cmp_key.cache_key): # we got the token. i'll get it from db while other threads wait! value = self.lpm[cmp_key.original_key_obj] self.cache.set(cmp_key.cache_key, value) return value # someone is already getting it time.sleep(1) # wait a little, and get it again (hopefully from cache) return self.__getitem__(cmp_key.original_key_obj) def delete(self, key): "Deletes a key-value map from memcached and dynamodb. Ignores it if item does not exist" Signals.pre_update.send(sender=self.__class__, map_name=self.name, key=key, action='delete') cmp_key = LPCMKey(self.name, key) self.cache.delete(cmp_key.cache_key) self.lpm.delete(key) Signals.post_update.send(sender=self.__class__, map_name=self.name, key=key, action='delete') def increment(self, key, value=1): if not isinstance(value, numbers.Number): raise ValueError( "Invalid increment value: {}. Only numbers are supported". format(value)) self._atomic_add_value(key, value) def decrement(self, key, value=1): if not isinstance(value, numbers.Number): raise ValueError( "Invalid decrement value: {}. Only numbers are supported". format(value)) self._atomic_add_value(key, value * -1) def _atomic_add_value(self, key, value): Signals.pre_update.send(sender=self.__class__, map_name=self.name, key=key, action='put') cmp_key = LPCMKey(self.name, key) self.lpm.atomic_add_value(key, value) self.cache.delete(cmp_key.cache_key) Signals.post_update.send(sender=self.__class__, map_name=self.name, key=key, action='put') def _atomic_delete_values(self, key, values): Signals.pre_update.send(sender=self.__class__, map_name=self.name, key=key, action='put') cmp_key = LPCMKey(self.name, key) self.lpm.atomic_delete_values(key, values) self.cache.delete(cmp_key.cache_key) Signals.post_update.send(sender=self.__class__, map_name=self.name, key=key, action='put') def __iter__(self): """ Note: this method is EXPENSIVE! PLease only use if absolutely needed""" return self.lpm.__iter__() def items(self): """ Note: this method is EXPENSIVE! PLease only use if absolutely needed""" return [(k, self[k]) for k in self] def keys(self): """ Note: this method is EXPENSIVE! PLease only use if absoluely needed""" return [k for k in self] def pop(self, k, d=None): """ D.pop(k[,d]) -> v, remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise KeyError is raised """ pass def clear(self): raise NotImplementedError() def copy(self): raise NotImplementedError()
def __init__(self, name, cache_timeout = None): "When cache_only is true, the data will be stored in memcached only (used in tests)" self.name = name self.lpm = LargePersistentMap(name) self.cache = Cache(cache_timeout)
class LargePersistentCachedMap(object): """ A key:value dictionary which is saved both in Memcached and AWS DynamoDB. Create a map by simply calling the constructor with the map name. Supported value types are strings and numbers. Maximum value size is 60kb. m = LargePersistentCachedMap("my_table") m["key1"] = "some string" m["key2"] = set(["d", "e"])31415 m2 = LargePersistentCachedMap("my_table") print m2["key1"] """ def __init__(self, name, cache_timeout = None): "When cache_only is true, the data will be stored in memcached only (used in tests)" self.name = name self.lpm = LargePersistentMap(name) self.cache = Cache(cache_timeout) def __setitem__(self, key, value): Signals.pre_update.send(sender = self.__class__, map_name = self.name, key = key, action = 'put') cmp_key = LPCMKey(self.name, key) self.cache.set(cmp_key.cache_key, value) self.lpm[key] = value Signals.post_update.send(sender = self.__class__, map_name = self.name, key = key, action = 'put') def __getitem__(self, key): cmp_key = LPCMKey(self.name, key) v = self.cache.get(cmp_key.cache_key) if v is not None: return v return self._get_from_dynamodb_and_save_in_cache(cmp_key) def get(self, k, d = None): """ D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None. """ try: return self[k] except KeyError: return d def __contains__(self, key): try: v = self[key] except KeyError: return False return True def disable_caching(self): self.cache = CacheDisabled() def _get_from_dynamodb_and_save_in_cache(self, cmp_key): """ Gets value from dynamodb and puts it in cache, taking care of the cache-miss stampede """ if self.cache.get_thread_safe_token(cmp_key.cache_key): # we got the token. i'll get it from db while other threads wait! value = self.lpm[cmp_key.original_key_obj] self.cache.set(cmp_key.cache_key, value) return value # someone is already getting it time.sleep(1) # wait a little, and get it again (hopefully from cache) return self.__getitem__(cmp_key.original_key_obj) def delete(self, key): "Deletes a key-value map from memcached and dynamodb. Ignores it if item does not exist" Signals.pre_update.send(sender = self.__class__, map_name = self.name, key = key, action = 'delete') cmp_key = LPCMKey(self.name, key) self.cache.delete(cmp_key.cache_key) self.lpm.delete(key) Signals.post_update.send(sender = self.__class__, map_name = self.name, key = key, action = 'delete') def increment(self, key, value = 1): if not isinstance(value, numbers.Number): raise ValueError( "Invalid increment value: {}. Only numbers are supported".format(value)) self._atomic_add_value(key, value) def decrement(self, key, value = 1): if not isinstance(value, numbers.Number): raise ValueError( "Invalid decrement value: {}. Only numbers are supported".format(value)) self._atomic_add_value(key, value * -1) def _atomic_add_value(self, key, value): Signals.pre_update.send(sender = self.__class__, map_name = self.name, key = key, action = 'put') cmp_key = LPCMKey(self.name, key) self.lpm.atomic_add_value(key, value) self.cache.delete(cmp_key.cache_key) Signals.post_update.send(sender = self.__class__, map_name = self.name, key = key, action = 'put') def _atomic_delete_values(self, key, values): Signals.pre_update.send(sender = self.__class__, map_name = self.name, key = key, action = 'put') cmp_key = LPCMKey(self.name, key) self.lpm.atomic_delete_values(key, values) self.cache.delete(cmp_key.cache_key) Signals.post_update.send(sender = self.__class__, map_name = self.name, key = key, action = 'put') def __iter__(self): """ Note: this method is EXPENSIVE! PLease only use if absolutely needed""" return self.lpm.__iter__() def items(self): """ Note: this method is EXPENSIVE! PLease only use if absolutely needed""" return [(k, self[k]) for k in self] def keys(self): """ Note: this method is EXPENSIVE! PLease only use if absoluely needed""" return [k for k in self] def pop(self, k, d = None): """ D.pop(k[,d]) -> v, remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise KeyError is raised """ pass def clear(self): raise NotImplementedError() def copy(self): raise NotImplementedError()