def __init__(self, sqluri, standard_collections=False, fixed_collections=False, use_quota=False, quota_size=0, pool_size=100, pool_recycle=3600, cache_servers=None, mirrored_cache_servers=None, create_tables=False, shard=False, shardsize=100, memcached_json=False, **kw): self.sqlstorage = super(MemcachedSQLStorage, self) self.sqlstorage.__init__(sqluri, standard_collections, fixed_collections, use_quota, quota_size, pool_size, pool_recycle, create_tables=create_tables, shard=shard, shardsize=shardsize, **kw) if isinstance(cache_servers, str): cache_servers = [cache_servers] elif cache_servers is None: cache_servers = ['127.0.0.1:11211'] if isinstance(mirrored_cache_servers, str): mirrored_cache_servers = [mirrored_cache_servers] extra_kw = {} if memcached_json: extra_kw['pickler'] = _JSONDumper extra_kw['unpickler'] = _JSONDumper if mirrored_cache_servers is None: self.cache = CacheManager(cache_servers, **extra_kw) else: self.cache = MirroredCacheManager(cache_servers, mirrored_cache_servers, **extra_kw)
def __init__(self, sqluri, standard_collections=False, fixed_collections=False, use_quota=False, quota_size=0, pool_size=100, pool_recycle=3600, cache_servers=None, mirrored_cache_servers=None, create_tables=False, shard=False, shardsize=100, memcached_json=False, **kw): self.sqlstorage = super(MemcachedSQLStorage, self) self.sqlstorage.__init__(sqluri, standard_collections, fixed_collections, use_quota, quota_size, pool_size, pool_recycle, create_tables=create_tables, shard=shard, shardsize=shardsize, **kw) if isinstance(cache_servers, str): cache_servers = [cache_servers] elif cache_servers is None: cache_servers = ['127.0.0.1:11211'] if isinstance(mirrored_cache_servers, str): mirrored_cache_servers = [mirrored_cache_servers] extra_kw = {} if memcached_json: extra_kw['pickler'] = _JSONDumper extra_kw['unpickler'] = _JSONDumper if mirrored_cache_servers is None: self.cache = CacheManager(cache_servers, **extra_kw) else: self.cache = MirroredCacheManager(cache_servers, mirrored_cache_servers, **extra_kw)
class MemcachedSQLStorage(SQLStorage): """Uses Memcached when possible/useful, SQL otherwise. """ def __init__(self, sqluri, standard_collections=False, fixed_collections=False, use_quota=False, quota_size=0, pool_size=100, pool_recycle=3600, cache_servers=None, mirrored_cache_servers=None, create_tables=False, shard=False, shardsize=100, memcached_json=False, **kw): self.sqlstorage = super(MemcachedSQLStorage, self) self.sqlstorage.__init__(sqluri, standard_collections, fixed_collections, use_quota, quota_size, pool_size, pool_recycle, create_tables=create_tables, shard=shard, shardsize=shardsize, **kw) if isinstance(cache_servers, str): cache_servers = [cache_servers] elif cache_servers is None: cache_servers = ['127.0.0.1:11211'] if isinstance(mirrored_cache_servers, str): mirrored_cache_servers = [mirrored_cache_servers] extra_kw = {} if memcached_json: extra_kw['pickler'] = _JSONDumper extra_kw['unpickler'] = _JSONDumper if mirrored_cache_servers is None: self.cache = CacheManager(cache_servers, **extra_kw) else: self.cache = MirroredCacheManager(cache_servers, mirrored_cache_servers, **extra_kw) @classmethod def get_name(self): return 'memcached' # # Cache management # def _is_meta_global(self, collection_name, item_id): return collection_name == 'meta' and item_id == 'global' # # Cached APIs # def delete_storage(self, user_id): self.cache.flush_user_cache(user_id) self.sqlstorage.delete_storage(user_id) def delete_user(self, user_id): self.cache.flush_user_cache(user_id) self.sqlstorage.delete_user(user_id) def item_exists(self, user_id, collection_name, item_id): """Returns a timestamp if an item exists.""" def _item_exists(): return self.sqlstorage.item_exists(user_id, collection_name, item_id) # returning cached values when possible if self._is_meta_global(collection_name, item_id): key = _key(user_id, 'meta', 'global') wbo = self.cache.get(key) if wbo is not None: return wbo['modified'] # going sql.. elif collection_name == 'tabs': return self.cache.tab_exists(user_id, item_id) return self.sqlstorage.item_exists(user_id, collection_name, item_id) def get_items(self, user_id, collection_name, fields=None, filters=None, limit=None, offset=None, sort=None): """returns items from a collection "filter" is a dict used to add conditions to the db query. Its keys are the field names on which the condition operates. Its values are the values the field should have. It can be a single value, or a list. For the latter the in() operator is used. For single values, the operator has to be provided. """ # returning cached values when possible if collection_name == 'tabs': # tabs are not stored at all in SQL return self.cache.get_tabs(user_id, filters).values() return self.sqlstorage.get_items(user_id, collection_name, fields, filters, limit, offset, sort) def get_item(self, user_id, collection_name, item_id, fields=None): """Returns one item. If the item is meta/global, we want to get the cached one if present. """ def _get_item(): return self.sqlstorage.get_item(user_id, collection_name, item_id, fields) # returning cached values when possible if self._is_meta_global(collection_name, item_id): key = _key(user_id, 'meta', 'global') return self.cache.get_set(key, _get_item) elif collection_name == 'tabs': # tabs are not stored at all in SQL return self.cache.get_tab(user_id, item_id) return _get_item() def _update_stamp(self, user_id, collection_name, storage_time): # update the stamps cache if storage_time is None: storage_time = round_time() stamps = self.get_collection_timestamps(user_id) stamps[collection_name] = storage_time self.cache.set(_key(user_id, 'stamps'), stamps) def _update_cache(self, user_id, collection_name, items, storage_time): # update the total size cache (bytes) total_size = sum([len(item.get('payload', '')) for item in items]) self.cache.incr(_key(user_id, 'size'), total_size) # update the stamps cache self._update_stamp(user_id, collection_name, storage_time) # update the meta/global cache or the tabs cache if self._is_meta_global(collection_name, items[0]['id']): item = items[0] item['username'] = user_id key = _key(user_id, 'meta', 'global') self.cache.set(key, item) elif collection_name == 'tabs': tabs = dict([(item['id'], item) for item in items]) self.cache.set_tabs(user_id, tabs) def _update_item(self, item, when): if 'payload' in item and 'modified' not in item: item['modified'] = when def set_item(self, user_id, collection_name, item_id, storage_time=None, **values): """Adds or update an item""" values['id'] = item_id if storage_time is None: storage_time = round_time() self._update_item(values, storage_time) self._update_cache(user_id, collection_name, [values], storage_time) if collection_name == 'tabs': # return now : we don't store tabs in sql return storage_time return self.sqlstorage.set_item(user_id, collection_name, item_id, storage_time=storage_time, **values) def set_items(self, user_id, collection_name, items, storage_time=None): """Adds or update a batch of items. Returns a list of success or failures. """ if storage_time is None: storage_time = round_time() for item in items: self._update_item(item, storage_time) self._update_cache(user_id, collection_name, items, storage_time) if collection_name == 'tabs': # return now : we don't store tabs in sql return len(items) return self.sqlstorage.set_items(user_id, collection_name, items, storage_time=storage_time) def delete_item(self, user_id, collection_name, item_id, storage_time=None): """Deletes an item""" # Since we don't know how large the item is, we can't make any # useful adjustment to the cached size. Just leave it alone and # rely on automatic recalculation to keep it semi-accurate. # update the meta/global cache or the tabs cache if self._is_meta_global(collection_name, item_id): key = _key(user_id, 'meta', 'global') self.cache.delete(key) elif collection_name == 'tabs': # tabs are not stored at all in SQL if self.cache.delete_tab(user_id, item_id): self._update_stamp(user_id, 'tabs', storage_time) return True return False res = self.sqlstorage.delete_item(user_id, collection_name, item_id) if res: self._update_stamp(user_id, collection_name, storage_time) return res def delete_items(self, user_id, collection_name, item_ids=None, filters=None, limit=None, offset=None, sort=None, storage_time=None): """Deletes items. All items are removed unless item_ids is provided""" # Since we don't know how large the items are, we can't make any # useful adjustment to the cached size. Just leave it alone and # rely on automatic recalculation to keep it semi-accurate. # remove the cached values if (collection_name == 'meta' and (item_ids is None or 'global' in item_ids)): key = _key(user_id, 'meta', 'global') self.cache.delete(key) elif collection_name == 'tabs': # tabs are not stored at all in SQL if self.cache.delete_tabs(user_id, filters): self._update_stamp(user_id, 'tabs', storage_time) return True return False res = self.sqlstorage.delete_items(user_id, collection_name, item_ids, filters, limit, offset, sort) if res: self._update_stamp(user_id, collection_name, storage_time) return res def get_total_size(self, user_id, recalculate=False): """Returns the total size in KB of a user storage""" def _get_set_size(): # returns in KB size = self.sqlstorage.get_total_size(user_id) # adding the tabs size += self.cache.get_tabs_size(user_id) # update the cache and timestamp self.cache.set_total(user_id, size) self.cache.set(_key(user_id, "size", "ts"), int(time.time())) return size # Recalculate from the DB if requested, and if we haven't # already done so recently. if recalculate: last_recalc = self.cache.get(_key(user_id, "size", "ts")) if last_recalc is None: return _get_set_size() if time.time() - last_recalc > QUOTA_RECALCULATION_PERIOD: return _get_set_size() size = self.cache.get_total(user_id) if not size: # memcached server seems down or needs a reset return _get_set_size() return size def get_size_left(self, user_id, recalculate=False): """Returns the storage left for a user""" # This gets called to check quota on every write operation, # so don't recalculate from the database unless explicitly asked. if recalculate: size = self.get_total_size(user_id, recalculate) else: size = self.cache.get_total(user_id) if not size: size = 0 return self.quota_size - size def get_collection_sizes(self, user_id): """Returns the total size in KB for each collection of a user storage. """ # these sizes are not cached sizes = self.sqlstorage.get_collection_sizes(user_id) sizes['tabs'] = self.cache.get_tabs_size(user_id) # we can update the size while we're there, in case it's empty self.cache.set_total(user_id, sum(sizes.values())) return sizes def get_collection_timestamps(self, user_id): """Returns a cached version of the stamps when possible""" stamps = self.cache.get(_key(user_id, 'stamps')) # not cached yet or memcached is down if stamps is None: stamps = super(MemcachedSQLStorage, self).get_collection_timestamps(user_id) # adding the tabs stamp tabs_stamps = self.cache.get_tabs_timestamp(user_id) if tabs_stamps is not None: stamps['tabs'] = tabs_stamps # caching it self.cache.set(_key(user_id, 'stamps'), stamps) return stamps def get_collection_max_timestamp(self, user_id, collection_name): # let's get them all, so they get cached stamps = self.get_collection_timestamps(user_id) if collection_name == 'tabs' and 'tabs' not in stamps: return None return stamps.get(collection_name)
class MemcachedSQLStorage(SQLStorage): """Uses Memcached when possible/useful, SQL otherwise. """ def __init__(self, sqluri, standard_collections=False, fixed_collections=False, use_quota=False, quota_size=0, pool_size=100, pool_recycle=3600, cache_servers=None, mirrored_cache_servers=None, create_tables=False, shard=False, shardsize=100, memcached_json=False, **kw): self.sqlstorage = super(MemcachedSQLStorage, self) self.sqlstorage.__init__(sqluri, standard_collections, fixed_collections, use_quota, quota_size, pool_size, pool_recycle, create_tables=create_tables, shard=shard, shardsize=shardsize, **kw) if isinstance(cache_servers, str): cache_servers = [cache_servers] elif cache_servers is None: cache_servers = ['127.0.0.1:11211'] if isinstance(mirrored_cache_servers, str): mirrored_cache_servers = [mirrored_cache_servers] extra_kw = {} if memcached_json: extra_kw['pickler'] = _JSONDumper extra_kw['unpickler'] = _JSONDumper if mirrored_cache_servers is None: self.cache = CacheManager(cache_servers, **extra_kw) else: self.cache = MirroredCacheManager(cache_servers, mirrored_cache_servers, **extra_kw) @classmethod def get_name(self): return 'memcached' # # Cache management # def _is_meta_global(self, collection_name, item_id): return collection_name == 'meta' and item_id == 'global' # # Cached APIs # def delete_storage(self, user_id): self.cache.flush_user_cache(user_id) self.sqlstorage.delete_storage(user_id) def delete_user(self, user_id): self.cache.flush_user_cache(user_id) self.sqlstorage.delete_user(user_id) def item_exists(self, user_id, collection_name, item_id): """Returns a timestamp if an item exists.""" def _item_exists(): return self.sqlstorage.item_exists(user_id, collection_name, item_id) # returning cached values when possible if self._is_meta_global(collection_name, item_id): key = _key(user_id, 'meta', 'global') wbo = self.cache.get(key) if wbo is not None: return wbo['modified'] # going sql.. elif collection_name == 'tabs': return self.cache.tab_exists(user_id, item_id) return self.sqlstorage.item_exists(user_id, collection_name, item_id) def get_items(self, user_id, collection_name, fields=None, filters=None, limit=None, offset=None, sort=None): """returns items from a collection "filter" is a dict used to add conditions to the db query. Its keys are the field names on which the condition operates. Its values are the values the field should have. It can be a single value, or a list. For the latter the in() operator is used. For single values, the operator has to be provided. """ # returning cached values when possible if collection_name == 'tabs': # tabs are not stored at all in SQL return self.cache.get_tabs(user_id, filters).values() return self.sqlstorage.get_items(user_id, collection_name, fields, filters, limit, offset, sort) def get_item(self, user_id, collection_name, item_id, fields=None): """Returns one item. If the item is meta/global, we want to get the cached one if present. """ def _get_item(): return self.sqlstorage.get_item(user_id, collection_name, item_id, fields) # returning cached values when possible if self._is_meta_global(collection_name, item_id): key = _key(user_id, 'meta', 'global') return self.cache.get_set(key, _get_item) elif collection_name == 'tabs': # tabs are not stored at all in SQL return self.cache.get_tab(user_id, item_id) return _get_item() def _update_stamp(self, user_id, collection_name, storage_time): # update the stamps cache if storage_time is None: storage_time = round_time() stamps = self.get_collection_timestamps(user_id) stamps[collection_name] = storage_time self.cache.set(_key(user_id, 'stamps'), stamps) def _update_cache(self, user_id, collection_name, items, storage_time): # update the total size cache (bytes) total_size = sum([len(item.get('payload', '')) for item in items]) self.cache.incr(_key(user_id, 'size'), total_size) # update the stamps cache self._update_stamp(user_id, collection_name, storage_time) # update the meta/global cache or the tabs cache if self._is_meta_global(collection_name, items[0]['id']): item = items[0] item['username'] = user_id key = _key(user_id, 'meta', 'global') self.cache.set(key, item) elif collection_name == 'tabs': tabs = dict([(item['id'], item) for item in items]) self.cache.set_tabs(user_id, tabs) def _update_item(self, item, when): if 'payload' in item and 'modified' not in item: item['modified'] = when def set_item(self, user_id, collection_name, item_id, storage_time=None, **values): """Adds or update an item""" values['id'] = item_id if storage_time is None: storage_time = round_time() self._update_item(values, storage_time) self._update_cache(user_id, collection_name, [values], storage_time) if collection_name == 'tabs': # return now : we don't store tabs in sql return storage_time return self.sqlstorage.set_item(user_id, collection_name, item_id, storage_time=storage_time, **values) def set_items(self, user_id, collection_name, items, storage_time=None): """Adds or update a batch of items. Returns a list of success or failures. """ if storage_time is None: storage_time = round_time() for item in items: self._update_item(item, storage_time) self._update_cache(user_id, collection_name, items, storage_time) if collection_name == 'tabs': # return now : we don't store tabs in sql return len(items) return self.sqlstorage.set_items(user_id, collection_name, items, storage_time=storage_time) def delete_item(self, user_id, collection_name, item_id, storage_time=None): """Deletes an item""" # Since we don't know how large the item is, we can't make any # useful adjustment to the cached size. Just leave it alone and # rely on automatic recalculation to keep it semi-accurate. # update the meta/global cache or the tabs cache if self._is_meta_global(collection_name, item_id): key = _key(user_id, 'meta', 'global') self.cache.delete(key) elif collection_name == 'tabs': # tabs are not stored at all in SQL if self.cache.delete_tab(user_id, item_id): self._update_stamp(user_id, 'tabs', storage_time) return True return False res = self.sqlstorage.delete_item(user_id, collection_name, item_id) if res: self._update_stamp(user_id, collection_name, storage_time) return res def delete_items(self, user_id, collection_name, item_ids=None, filters=None, limit=None, offset=None, sort=None, storage_time=None): """Deletes items. All items are removed unless item_ids is provided""" # Since we don't know how large the items are, we can't make any # useful adjustment to the cached size. Just leave it alone and # rely on automatic recalculation to keep it semi-accurate. # remove the cached values if (collection_name == 'meta' and (item_ids is None or 'global' in item_ids)): key = _key(user_id, 'meta', 'global') self.cache.delete(key) elif collection_name == 'tabs': # tabs are not stored at all in SQL if self.cache.delete_tabs(user_id, filters): self._update_stamp(user_id, 'tabs', storage_time) return True return False res = self.sqlstorage.delete_items(user_id, collection_name, item_ids, filters, limit, offset, sort) if res: self._update_stamp(user_id, collection_name, storage_time) return res def get_total_size(self, user_id, recalculate=False): """Returns the total size in KB of a user storage""" def _get_set_size(): # returns in KB size = self.sqlstorage.get_total_size(user_id) # adding the tabs size += self.cache.get_tabs_size(user_id) # update the cache and timestamp self.cache.set_total(user_id, size) self.cache.set(_key(user_id, "size", "ts"), int(time.time())) return size # Recalculate from the DB if requested, and if we haven't # already done so recently. if recalculate: last_recalc = self.cache.get(_key(user_id, "size", "ts")) if last_recalc is None: return _get_set_size() if time.time() - last_recalc > QUOTA_RECALCULATION_PERIOD: return _get_set_size() size = self.cache.get_total(user_id) if not size: # memcached server seems down or needs a reset return _get_set_size() return size def get_size_left(self, user_id, recalculate=False): """Returns the storage left for a user""" # This gets called to check quota on every write operation, # so don't recalculate from the database unless explicitly asked. if recalculate: size = self.get_total_size(user_id, recalculate) else: size = self.cache.get_total(user_id) if not size: size = 0 return self.quota_size - size def get_collection_sizes(self, user_id): """Returns the total size in KB for each collection of a user storage. """ # these sizes are not cached sizes = self.sqlstorage.get_collection_sizes(user_id) sizes['tabs'] = self.cache.get_tabs_size(user_id) # we can update the size while we're there, in case it's empty self.cache.set_total(user_id, sum(sizes.values())) return sizes def get_collection_timestamps(self, user_id): """Returns a cached version of the stamps when possible""" stamps = self.cache.get(_key(user_id, 'stamps')) # not cached yet or memcached is down if stamps is None: stamps = super(MemcachedSQLStorage, self).get_collection_timestamps(user_id) # adding the tabs stamp tabs_stamps = self.cache.get_tabs_timestamp(user_id) if tabs_stamps is not None: stamps['tabs'] = tabs_stamps # caching it self.cache.set(_key(user_id, 'stamps'), stamps) return stamps def get_collection_max_timestamp(self, user_id, collection_name): # let's get them all, so they get cached stamps = self.get_collection_timestamps(user_id) if collection_name == 'tabs' and 'tabs' not in stamps: return None return stamps.get(collection_name)