def delete_user(self, user):
if not self.is_owner(user):
return False
colls_table = self._get_user_colls(user)
for coll in colls_table:
self.delete_collection(user, coll)
# user directory
user_dir = self.path_router.get_user_account_root(user)
rel_dir = os.path.relpath(user_dir, self.path_router.root_dir) + '/'
# delete from remote storage
self.storage_manager.delete_dir(rel_dir)
user_key = self._user_key(user)
with utils.pipeline(self.redis) as pi:
pi.delete(user_key)
pi.delete(user_key + self.COLL_KEY)
pi.publish('delete_user', user_dir)
# delete from cork!
self.cork.user(user).delete()
return True
def delete_user(self, user):
if not self.is_owner(user):
return False
colls_table = self._get_user_colls(user)
for coll in colls_table:
self.delete_collection(user, coll)
# user directory
user_dir = self.path_router.get_user_account_root(user)
rel_dir = os.path.relpath(user_dir, self.path_router.root_dir) + '/'
# delete from remote storage
self.storage_manager.delete_dir(rel_dir)
user_key = self._user_key(user)
with utils.pipeline(self.redis) as pi:
pi.delete(user_key)
pi.delete(user_key + self.COLL_KEY)
pi.publish('delete_user', user_dir)
# delete from cork!
self.cork.user(user).delete()
return True
def delete_collection(self, user, coll):
if not self.can_admin_coll(user, coll):
return False
base_key = self.make_key(user, coll)
keys = map(lambda x: base_key + x, self.ALL_KEYS)
#key_q = '*' + user + ':' + coll + '*'
#keys = self.redis.keys(key_q)
coll_len = self.redis.hget(base_key + self.DEDUP_KEY, 'total_len')
user_key = self._user_key(user)
user_len = self.redis.hget(user_key, 'total_len')
with utils.pipeline(self.redis) as pi:
if coll_len:
pi.hset(user_key, 'total_len', int(user_len) - int(coll_len))
# delete all coll keys
if keys:
pi.delete(*keys)
# send delete msg
pi.publish('delete_coll', self.path_router.get_archive_dir(user, coll))
# coll directory
coll_dir = self.path_router.get_coll_root(user, coll)
rel_dir = os.path.relpath(coll_dir, self.path_router.root_dir) + '/'
# delete from remote storage
self.storage_manager.delete_dir(rel_dir)
# delete collection entry
del self._get_user_colls(user)[coll]
return True
def delete_anon_user(self, user):
self.delete_collection(user, '@anon')
user_dir = self.path_router.get_user_account_root(user)
user_key = self._user_key(user)
with utils.pipeline(self.redis) as pi:
pi.delete(user_key)
pi.delete(user_key + self.COLL_KEY)
pi.publish('delete_user', user_dir)
return True
def delete_anon_user(self, user):
self.delete_collection(user, '@anon')
user_dir = self.path_router.get_user_account_root(user)
user_key = self._user_key(user)
with utils.pipeline(self.redis) as pi:
pi.delete(user_key)
pi.delete(user_key + self.COLL_KEY)
pi.publish('delete_user', user_dir)
return True
def archive_page():
browser_type, archive, url = get_params()
response_key = get_cache_key(archive, browser_type, url)
wait_key = get_wait_key(archive, browser_type, url)
queue_key = get_queue_key(browser_type)
result = None
if not rc.exists(response_key):
cmd = dict(request.query)
cmd['url'] = url
num = rc.incr('total_urls:' + browser_type)
cmd['num'] = num
cmd = json.dumps(cmd)
with pipeline(rc) as pi:
waiting_str = {'archived': False,
'queued': True,
'num': num}
pi.set(response_key, json.dumps(waiting_str))
pi.rpush(queue_key, cmd)
rc.blpop(wait_key, theconfig['wait_timeout_secs'])
result = rc.get(response_key)
if result:
result = json.loads(result)
if 'queued' in result:
result['queue_pos'] = 0
front = rc.lindex(queue_key, 0)
if front:
front = json.loads(front)
front_num = front.get('num', 0)
# pos == 1 implies this url is next up
# pos <= 0 implies this url was removed from queue and is being processed
pos = result['num'] - front_num + 1
result['queue_pos'] = pos
else:
result['ttl'] = rc.ttl(response_key)
else:
result = ERROR_RESP
return result
def archive_page():
browser_type, archive, url = get_params()
response_key = get_cache_key(archive, browser_type, url)
wait_key = get_wait_key(archive, browser_type, url)
queue_key = get_queue_key(browser_type)
result = None
if not rc.exists(response_key):
cmd = dict(request.query)
cmd['url'] = url
num = rc.incr('total_urls:' + browser_type)
cmd['num'] = num
cmd = json.dumps(cmd)
with pipeline(rc) as pi:
waiting_str = {'archived': False, 'queued': True, 'num': num}
pi.set(response_key, json.dumps(waiting_str))
pi.rpush(queue_key, cmd)
rc.blpop(wait_key, theconfig['wait_timeout_secs'])
result = rc.get(response_key)
if result:
result = json.loads(result)
if 'queued' in result:
result['queue_pos'] = 0
front = rc.lindex(queue_key, 0)
if front:
front = json.loads(front)
front_num = front.get('num', 0)
# pos == 1 implies this url is next up
# pos <= 0 implies this url was removed from queue and is being processed
pos = result['num'] - front_num + 1
result['queue_pos'] = pos
else:
result['ttl'] = rc.ttl(response_key)
else:
result = ERROR_RESP
return result
def run(rc, browser, archives, config, browser_type):
""" Read from redis queue in a loop and use associated web driver
to load page on demand
"""
url = None
queue_key = get_queue_key(browser_type)
logging.debug(queue_key)
while True:
cmd = rc.blpop(queue_key, 10)
if not cmd:
continue
val = json.loads(cmd[1])
archive = val['archive']
url = val['url']
result_key = get_cache_key(archive, browser_type, url)
wait_key = get_wait_key(archive, browser_type, url)
try:
result = archives[archive](browser, url)
cache_time = config['archive_cache_secs']
except Exception as e:
import traceback
traceback.print_exc()
result = {'archived': False, 'error': {'msg': str(e)}}
cache_time = config['err_cache_secs']
json_result = json.dumps(result)
actual_url = result.get('actual_url')
with pipeline(rc) as pi:
if actual_url and actual_url != url:
actual_key = get_cache_key(archive, browser_type, actual_url)
pi.setex(actual_key, cache_time, json_result)
pi.setex(result_key, cache_time, json_result)
pi.rpush(wait_key, 1)
pi.expire(wait_key, cache_time)
def run(rc, browser, archives, config, browser_type):
""" Read from redis queue in a loop and use associated web driver
to load page on demand
"""
url = None
queue_key = get_queue_key(browser_type)
logging.debug(queue_key)
while True:
cmd = rc.blpop(queue_key, 10)
if not cmd:
continue
val= json.loads(cmd[1])
archive = val['archive']
url = val['url']
result_key = get_cache_key(archive, browser_type, url)
wait_key = get_wait_key(archive, browser_type, url)
try:
result = archives[archive](browser, url)
cache_time = config['archive_cache_secs']
except Exception as e:
import traceback
traceback.print_exc()
result = {'archived': False, 'error': {'msg': str(e) }}
cache_time = config['err_cache_secs']
json_result = json.dumps(result)
actual_url = result.get('actual_url')
with pipeline(rc) as pi:
if actual_url and actual_url != url:
actual_key = get_cache_key(archive, browser_type, actual_url)
pi.setex(actual_key, cache_time, json_result)
pi.setex(result_key, cache_time, json_result)
pi.rpush(wait_key, 1)
pi.expire(wait_key, cache_time)
def delete_collection(self, user, coll):
if not self.can_admin_coll(user, coll):
return False
base_key = self.make_key(user, coll)
keys = map(lambda x: base_key + x, self.ALL_KEYS)
#key_q = '*' + user + ':' + coll + '*'
#keys = self.redis.keys(key_q)
coll_len = self.redis.hget(base_key + self.DEDUP_KEY, 'total_len')
user_key = self._user_key(user)
user_len = self.redis.hget(user_key, 'total_len')
with utils.pipeline(self.redis) as pi:
if coll_len:
pi.hset(user_key, 'total_len', int(user_len) - int(coll_len))
# delete all coll keys
if keys:
pi.delete(*keys)
# send delete msg
pi.publish('delete_coll',
self.path_router.get_archive_dir(user, coll))
# coll directory
coll_dir = self.path_router.get_coll_root(user, coll)
rel_dir = os.path.relpath(coll_dir, self.path_router.root_dir) + '/'
# delete from remote storage
self.storage_manager.delete_dir(rel_dir)
# delete collection entry
del self._get_user_colls(user)[coll]
return True
