def _update_stamp(self, user_id, collection_name, storage_time):
# update the stamps cache
if storage_time is None:
storage_time = round_time()
stamps = self.get_collection_timestamps(user_id)
stamps[collection_name] = storage_time
self.cache.set(_key(user_id, 'stamps'), stamps)
def delete_items(self, user_id, collection_name, item_ids=None,
filters=None, limit=None, offset=None, sort=None,
storage_time=None):
"""Deletes items. All items are removed unless item_ids is provided"""
# Since we don't know how large the items are, we can't make any
# useful adjustment to the cached size. Just leave it alone and
# rely on automatic recalculation to keep it semi-accurate.
# remove the cached values
if (collection_name == 'meta' and (item_ids is None
or 'global' in item_ids)):
key = _key(user_id, 'meta', 'global')
self.cache.delete(key)
elif collection_name == 'tabs':
# tabs are not stored at all in SQL
if self.cache.delete_tabs(user_id, filters):
self._update_stamp(user_id, 'tabs', storage_time)
return True
return False
res = self.sqlstorage.delete_items(user_id, collection_name,
item_ids, filters,
limit, offset, sort)
if res:
self._update_stamp(user_id, collection_name, storage_time)
return res
def delete_items(self,
user_id,
collection_name,
item_ids=None,
filters=None,
limit=None,
offset=None,
sort=None,
storage_time=None):
"""Deletes items. All items are removed unless item_ids is provided"""
# Since we don't know how large the items are, we can't make any
# useful adjustment to the cached size. Just leave it alone and
# rely on automatic recalculation to keep it semi-accurate.
# remove the cached values
if (collection_name == 'meta'
and (item_ids is None or 'global' in item_ids)):
key = _key(user_id, 'meta', 'global')
self.cache.delete(key)
elif collection_name == 'tabs':
# tabs are not stored at all in SQL
if self.cache.delete_tabs(user_id, filters):
self._update_stamp(user_id, 'tabs', storage_time)
return True
return False
res = self.sqlstorage.delete_items(user_id, collection_name, item_ids,
filters, limit, offset, sort)
if res:
self._update_stamp(user_id, collection_name, storage_time)
return res
def get_total_size(self, user_id, recalculate=False):
"""Returns the total size in KB of a user storage"""
def _get_set_size():
# returns in KB
size = self.sqlstorage.get_total_size(user_id)
# adding the tabs
size += self.cache.get_tabs_size(user_id)
# update the cache and timestamp
self.cache.set_total(user_id, size)
self.cache.set(_key(user_id, "size", "ts"), int(time.time()))
return size
# Recalculate from the DB if requested, and if we haven't
# already done so recently.
if recalculate:
last_recalc = self.cache.get(_key(user_id, "size", "ts"))
if last_recalc is None:
return _get_set_size()
if time.time() - last_recalc > QUOTA_RECALCULATION_PERIOD:
return _get_set_size()
size = self.cache.get_total(user_id)
if not size: # memcached server seems down or needs a reset
return _get_set_size()
return size
def delete_item(self,
user_id,
collection_name,
item_id,
storage_time=None):
"""Deletes an item"""
# Since we don't know how large the item is, we can't make any
# useful adjustment to the cached size. Just leave it alone and
# rely on automatic recalculation to keep it semi-accurate.
# update the meta/global cache or the tabs cache
if self._is_meta_global(collection_name, item_id):
key = _key(user_id, 'meta', 'global')
self.cache.delete(key)
elif collection_name == 'tabs':
# tabs are not stored at all in SQL
if self.cache.delete_tab(user_id, item_id):
self._update_stamp(user_id, 'tabs', storage_time)
return True
return False
res = self.sqlstorage.delete_item(user_id, collection_name, item_id)
if res:
self._update_stamp(user_id, collection_name, storage_time)
return res
def _update_stamp(self, user_id, collection_name, storage_time):
# update the stamps cache
if storage_time is None:
storage_time = round_time()
stamps = self.get_collection_timestamps(user_id)
stamps[collection_name] = storage_time
self.cache.set(_key(user_id, 'stamps'), stamps)
def get_total_size(self, user_id, recalculate=False):
"""Returns the total size in KB of a user storage"""
def _get_set_size():
# returns in KB
size = self.sqlstorage.get_total_size(user_id)
# adding the tabs
size += self.cache.get_tabs_size(user_id)
# update the cache and timestamp
self.cache.set_total(user_id, size)
self.cache.set(_key(user_id, "size", "ts"), int(time.time()))
return size
# Recalculate from the DB if requested, and if we haven't
# already done so recently.
if recalculate:
last_recalc = self.cache.get(_key(user_id, "size", "ts"))
if last_recalc is None:
return _get_set_size()
if time.time() - last_recalc > QUOTA_RECALCULATION_PERIOD:
return _get_set_size()
size = self.cache.get_total(user_id)
if not size: # memcached server seems down or needs a reset
return _get_set_size()
return size
def _update_cache(self, user_id, collection_name, items, storage_time):
# update the total size cache (bytes)
total_size = sum([len(item.get('payload', '')) for item in items])
self.cache.incr(_key(user_id, 'size'), total_size)
# update the stamps cache
self._update_stamp(user_id, collection_name, storage_time)
# update the meta/global cache or the tabs cache
if self._is_meta_global(collection_name, items[0]['id']):
item = items[0]
item['username'] = user_id
key = _key(user_id, 'meta', 'global')
self.cache.set(key, item)
elif collection_name == 'tabs':
tabs = dict([(item['id'], item) for item in items])
self.cache.set_tabs(user_id, tabs)
def _update_cache(self, user_id, collection_name, items, storage_time):
# update the total size cache (bytes)
total_size = sum([len(item.get('payload', '')) for item in items])
self.cache.incr(_key(user_id, 'size'), total_size)
# update the stamps cache
self._update_stamp(user_id, collection_name, storage_time)
# update the meta/global cache or the tabs cache
if self._is_meta_global(collection_name, items[0]['id']):
item = items[0]
item['username'] = user_id
key = _key(user_id, 'meta', 'global')
self.cache.set(key, item)
elif collection_name == 'tabs':
tabs = dict([(item['id'], item) for item in items])
self.cache.set_tabs(user_id, tabs)
def get_collection_timestamps(self, user_id):
"""Returns a cached version of the stamps when possible"""
stamps = self.cache.get(_key(user_id, 'stamps'))
# not cached yet or memcached is down
if stamps is None:
stamps = super(MemcachedSQLStorage,
self).get_collection_timestamps(user_id)
# adding the tabs stamp
tabs_stamps = self.cache.get_tabs_timestamp(user_id)
if tabs_stamps is not None:
stamps['tabs'] = tabs_stamps
# caching it
self.cache.set(_key(user_id, 'stamps'), stamps)
return stamps
def get_collection_timestamps(self, user_id):
"""Returns a cached version of the stamps when possible"""
stamps = self.cache.get(_key(user_id, 'stamps'))
# not cached yet or memcached is down
if stamps is None:
stamps = super(MemcachedSQLStorage,
self).get_collection_timestamps(user_id)
# adding the tabs stamp
tabs_stamps = self.cache.get_tabs_timestamp(user_id)
if tabs_stamps is not None:
stamps['tabs'] = tabs_stamps
# caching it
self.cache.set(_key(user_id, 'stamps'), stamps)
return stamps
def _get_set_size():
# returns in KB
size = self.sqlstorage.get_total_size(user_id)
# adding the tabs
size += self.cache.get_tabs_size(user_id)
# update the cache and timestamp
self.cache.set_total(user_id, size)
self.cache.set(_key(user_id, "size", "ts"), int(time.time()))
return size
def _get_set_size():
# returns in KB
size = self.sqlstorage.get_total_size(user_id)
# adding the tabs
size += self.cache.get_tabs_size(user_id)
# update the cache and timestamp
self.cache.set_total(user_id, size)
self.cache.set(_key(user_id, "size", "ts"), int(time.time()))
return size
def get_item(self, user_id, collection_name, item_id, fields=None):
"""Returns one item.
If the item is meta/global, we want to get the cached one if present.
"""
def _get_item():
return self.sqlstorage.get_item(user_id, collection_name,
item_id, fields)
# returning cached values when possible
if self._is_meta_global(collection_name, item_id):
key = _key(user_id, 'meta', 'global')
return self.cache.get_set(key, _get_item)
elif collection_name == 'tabs':
# tabs are not stored at all in SQL
return self.cache.get_tab(user_id, item_id)
return _get_item()
def get_item(self, user_id, collection_name, item_id, fields=None):
"""Returns one item.
If the item is meta/global, we want to get the cached one if present.
"""
def _get_item():
return self.sqlstorage.get_item(user_id, collection_name, item_id,
fields)
# returning cached values when possible
if self._is_meta_global(collection_name, item_id):
key = _key(user_id, 'meta', 'global')
return self.cache.get_set(key, _get_item)
elif collection_name == 'tabs':
# tabs are not stored at all in SQL
return self.cache.get_tab(user_id, item_id)
return _get_item()
def item_exists(self, user_id, collection_name, item_id):
"""Returns a timestamp if an item exists."""
def _item_exists():
return self.sqlstorage.item_exists(user_id, collection_name,
item_id)
# returning cached values when possible
if self._is_meta_global(collection_name, item_id):
key = _key(user_id, 'meta', 'global')
wbo = self.cache.get(key)
if wbo is not None:
return wbo['modified']
# going sql..
elif collection_name == 'tabs':
return self.cache.tab_exists(user_id, item_id)
return self.sqlstorage.item_exists(user_id, collection_name, item_id)
def item_exists(self, user_id, collection_name, item_id):
"""Returns a timestamp if an item exists."""
def _item_exists():
return self.sqlstorage.item_exists(user_id, collection_name,
item_id)
# returning cached values when possible
if self._is_meta_global(collection_name, item_id):
key = _key(user_id, 'meta', 'global')
wbo = self.cache.get(key)
if wbo is not None:
return wbo['modified']
# going sql..
elif collection_name == 'tabs':
return self.cache.tab_exists(user_id, item_id)
return self.sqlstorage.item_exists(user_id, collection_name, item_id)
def delete_item(self, user_id, collection_name, item_id,
storage_time=None):
"""Deletes an item"""
# Since we don't know how large the item is, we can't make any
# useful adjustment to the cached size. Just leave it alone and
# rely on automatic recalculation to keep it semi-accurate.
# update the meta/global cache or the tabs cache
if self._is_meta_global(collection_name, item_id):
key = _key(user_id, 'meta', 'global')
self.cache.delete(key)
elif collection_name == 'tabs':
# tabs are not stored at all in SQL
if self.cache.delete_tab(user_id, item_id):
self._update_stamp(user_id, 'tabs', storage_time)
return True
return False
res = self.sqlstorage.delete_item(user_id, collection_name, item_id)
if res:
self._update_stamp(user_id, collection_name, storage_time)
return res
