def __init__(self, sqluri, standard_collections=False, use_quota=False, quota_size=0, pool_size=100, pool_recycle=3600, cache_servers=None, create_tables=False, shard=False, shardsize=100, memcached_json=False, **kw): self.sqlstorage = super(MemcachedSQLStorage, self) self.sqlstorage.__init__(sqluri, standard_collections, use_quota, quota_size, pool_size, pool_recycle, create_tables=create_tables, shard=shard, shardsize=shardsize, **kw) if isinstance(cache_servers, str): cache_servers = [cache_servers] elif cache_servers is None: cache_servers = ['127.0.0.1:11211'] extra_kw = {} if memcached_json: extra_kw['pickler'] = _JSONDumper extra_kw['unpickler'] = _JSONDumper self.cache = CacheManager(cache_servers, **extra_kw)
def __init__(self, sqluri, standard_collections=False, fixed_collections=False, use_quota=False, quota_size=0, pool_size=100, pool_recycle=3600, cache_servers=None, mirrored_cache_servers=None, create_tables=False, shard=False, shardsize=100, memcached_json=False, **kw): self.sqlstorage = super(MemcachedSQLStorage, self) self.sqlstorage.__init__(sqluri, standard_collections, fixed_collections, use_quota, quota_size, pool_size, pool_recycle, create_tables=create_tables, shard=shard, shardsize=shardsize, **kw) if isinstance(cache_servers, str): cache_servers = [cache_servers] elif cache_servers is None: cache_servers = ['127.0.0.1:11211'] if isinstance(mirrored_cache_servers, str): mirrored_cache_servers = [mirrored_cache_servers] extra_kw = {} if memcached_json: extra_kw['pickler'] = _JSONDumper extra_kw['unpickler'] = _JSONDumper if mirrored_cache_servers is None: self.cache = CacheManager(cache_servers, **extra_kw) else: self.cache = MirroredCacheManager(cache_servers, mirrored_cache_servers, **extra_kw)
class MemcachedSQLStorage(SQLStorage): """Uses Memcached when possible/useful, SQL otherwise. """ def __init__(self, sqluri, standard_collections=False, use_quota=False, quota_size=0, pool_size=100, pool_recycle=3600, cache_servers=None, create_tables=False, shard=False, shardsize=100, memcached_json=False, **kw): self.sqlstorage = super(MemcachedSQLStorage, self) self.sqlstorage.__init__(sqluri, standard_collections, use_quota, quota_size, pool_size, pool_recycle, create_tables=create_tables, shard=shard, shardsize=shardsize, **kw) if isinstance(cache_servers, str): cache_servers = [cache_servers] elif cache_servers is None: cache_servers = ['127.0.0.1:11211'] extra_kw = {} if memcached_json: extra_kw['pickler'] = _JSONDumper extra_kw['unpickler'] = _JSONDumper self.cache = CacheManager(cache_servers, **extra_kw) @classmethod def get_name(self): return 'memcached' # # Cache management # def _is_meta_global(self, collection_name, item_id): return collection_name == 'meta' and item_id == 'global' # # Cached APIs # def delete_storage(self, user_id): self.cache.flush_user_cache(user_id) self.sqlstorage.delete_storage(user_id) def delete_user(self, user_id): self.cache.flush_user_cache(user_id) self.sqlstorage.delete_user(user_id) def item_exists(self, user_id, collection_name, item_id): """Returns a timestamp if an item exists.""" def _item_exists(): return self.sqlstorage.item_exists(user_id, collection_name, item_id) # returning cached values when possible if self._is_meta_global(collection_name, item_id): key = _key(user_id, 'meta', 'global') wbo = self.cache.get(key) if wbo is not None: return wbo['modified'] # going sql.. elif collection_name == 'tabs': return self.cache.tab_exists(user_id, item_id) return self.sqlstorage.item_exists(user_id, collection_name, item_id) def get_items(self, user_id, collection_name, fields=None, filters=None, limit=None, offset=None, sort=None): """returns items from a collection "filter" is a dict used to add conditions to the db query. Its keys are the field names on which the condition operates. Its values are the values the field should have. It can be a single value, or a list. For the latter the in() operator is used. For single values, the operator has to be provided. """ # returning cached values when possible if collection_name == 'tabs': # tabs are not stored at all in SQL return self.cache.get_tabs(user_id, filters).values() return self.sqlstorage.get_items(user_id, collection_name, fields, filters, limit, offset, sort) def get_item(self, user_id, collection_name, item_id, fields=None): """Returns one item. If the item is meta/global, we want to get the cached one if present. """ def _get_item(): return self.sqlstorage.get_item(user_id, collection_name, item_id, fields) # returning cached values when possible if self._is_meta_global(collection_name, item_id): key = _key(user_id, 'meta', 'global') return self.cache.get_set(key, _get_item) elif collection_name == 'tabs': # tabs are not stored at all in SQL return self.cache.get_tab(user_id, item_id) return _get_item() def _delete_cache(self, user_id): """Removes all cached data.""" for key in ('size', 'meta:global', 'tabs'): self.cache.delete(_key(user_id, key)) def _update_stamp(self, user_id, collection_name, storage_time): # update the stamps cache if storage_time is None: storage_time = round_time() stamps = self.get_collection_timestamps(user_id) stamps[collection_name] = storage_time self.cache.set(_key(user_id, 'stamps'), stamps) def _update_cache(self, user_id, collection_name, items, storage_time): # update the total size cache (bytes) total_size = sum([len(item.get('payload', '')) for item in items]) self.cache.incr(_key(user_id, 'size'), total_size) # update the stamps cache self._update_stamp(user_id, collection_name, storage_time) # update the meta/global cache or the tabs cache if self._is_meta_global(collection_name, items[0]['id']): item = items[0] item['username'] = user_id key = _key(user_id, 'meta', 'global') self.cache.set(key, item) elif collection_name == 'tabs': tabs = dict([(item['id'], item) for item in items]) self.cache.set_tabs(user_id, tabs) def _update_item(self, item, when): if 'payload' in item: item['modified'] = when def set_item(self, user_id, collection_name, item_id, storage_time=None, **values): """Adds or update an item""" values['id'] = item_id if storage_time is None: storage_time = round_time() self._update_item(values, storage_time) self._update_cache(user_id, collection_name, [values], storage_time) if collection_name == 'tabs': # return now : we don't store tabs in sql return storage_time return self.sqlstorage.set_item(user_id, collection_name, item_id, storage_time=storage_time, **values) def set_items(self, user_id, collection_name, items, storage_time=None): """Adds or update a batch of items. Returns a list of success or failures. """ if storage_time is None: storage_time = round_time() for item in items: self._update_item(item, storage_time) self._update_cache(user_id, collection_name, items, storage_time) if collection_name == 'tabs': # return now : we don't store tabs in sql return len(items) return self.sqlstorage.set_items(user_id, collection_name, items, storage_time=storage_time) def delete_item(self, user_id, collection_name, item_id, storage_time=None): """Deletes an item""" # delete the cached size - will be recalculated self.cache.delete(_key(user_id, 'size')) # update the meta/global cache or the tabs cache if self._is_meta_global(collection_name, item_id): key = _key(user_id, 'meta', 'global') self.cache.delete(key) elif collection_name == 'tabs': # tabs are not stored at all in SQL if self.cache.delete_tab(user_id, item_id): self._update_stamp(user_id, 'tabs', storage_time) return True return False res = self.sqlstorage.delete_item(user_id, collection_name, item_id) if res: self._update_stamp(user_id, collection_name, storage_time) return res def delete_items(self, user_id, collection_name, item_ids=None, filters=None, limit=None, offset=None, sort=None, storage_time=None): """Deletes items. All items are removed unless item_ids is provided""" # delete the cached size self.cache.delete(_key(user_id, 'size')) # remove the cached values if (collection_name == 'meta' and (item_ids is None or 'global' in item_ids)): key = _key(user_id, 'meta', 'global') self.cache.delete(key) elif collection_name == 'tabs': # tabs are not stored at all in SQL if self.cache.delete_tabs(user_id, filters): self._update_stamp(user_id, 'tabs', storage_time) return True return False res = self.sqlstorage.delete_items(user_id, collection_name, item_ids, filters, limit, offset, sort) if res: self._update_stamp(user_id, collection_name, storage_time) return res def _set_cached_size(self, user_id, size): key = _key(user_id, 'size') # if this fail it's not a big deal try: # we store the size in bytes in memcached self.cache.set(key, size * _KB) except BackendError: logger.error('Could not write to memcached') def _get_cached_size(self, user_id): try: size = self.cache.get(_key(user_id, 'size')) if size != 0 and size is not None: size = size / _KB except BackendError: size = None return size def get_total_size(self, user_id, recalculate=False): """Returns the total size in KB of a user storage""" def _get_set_size(): # returns in KB size = self.sqlstorage.get_total_size(user_id) # adding the tabs size += self.cache.get_tabs_size(user_id) # update the cache self.cache.set_total(user_id, size) return size if recalculate: return _get_set_size() size = self.cache.get_total(user_id) if not size: # memcached server seems down or needs a reset return _get_set_size() return size def get_collection_sizes(self, user_id): """Returns the total size in KB for each collection of a user storage. """ # these sizes are not cached sizes = self.sqlstorage.get_collection_sizes(user_id) sizes['tabs'] = self.cache.get_tabs_size(user_id) # we can update the size while we're there, in case it's empty self.cache.set_total(user_id, sum(sizes.values())) return sizes def get_collection_timestamps(self, user_id): """Returns a cached version of the stamps when possible""" stamps = self.cache.get(_key(user_id, 'stamps')) # not cached yet or memcached is down if stamps is None: stamps = super(MemcachedSQLStorage, self).get_collection_timestamps(user_id) # adding the tabs stamp tabs_stamps = self.cache.get_tabs_timestamp(user_id) if tabs_stamps is not None: stamps['tabs'] = tabs_stamps # caching it self.cache.set(_key(user_id, 'stamps'), stamps) return stamps def get_collection_max_timestamp(self, user_id, collection_name): # let's get them all, so they get cached stamps = self.get_collection_timestamps(user_id) if collection_name == 'tabs' and 'tabs' not in stamps: return None return stamps[collection_name]