def download_queue_processor():
"""
Implements a simple re-try mechanism for pending downloads
:return:
"""
while True:
if download_queue.not_empty:
item, path, oauth = download_queue.get() # blocks
if item['type'] == 'file':
info = redis_get(r_c, item) if r_c.exists(redis_key(item['id'])) else None
# client = Client(oauth) # keep it around for easy access
# hack because we did not use to store the file_path, but do not want to force a download
if info and 'file_path' not in info:
info['file_path'] = path
r_c.set(redis_key(item['id']), json.dumps(info))
r_c.set('diy_crate.last_save_time_stamp', int(time.time()))
# no version, or diff version, or the file does not exist locally
if not info or info['etag'] != item['etag'] or not os.path.exists(path):
try:
for i in range(15):
if os.path.basename(path).startswith('.~lock'): # avoid downloading lock files
break
try:
with open(path, 'wb') as item_handler:
crate_logger.debug('About to download: {obj_name}, '
'{obj_id}'.format(obj_name=item['name'], obj_id=item['id']))
item.download_to(item_handler)
path_to_add = os.path.dirname(path)
wm.add_watch(path=path_to_add, mask=mask, rec=True, auto_add=True)
notify_user_with_gui('Downloaded: {}'.format(path))
except BoxAPIException as e:
crate_logger.debug(traceback.format_exc())
if e.status == 404:
crate_logger.debug('Apparently item: {obj_id}, {path} has been deleted, '
'right before we tried to download'.format(obj_id=item['id'],
path=path))
break
was_versioned = r_c.exists(redis_key(item['id']))
#
# version_info[item['id']] = version_info.get(item['id'], {'etag': item['etag'],
# 'fresh_download': True,
# 'time_stamp': time.time()})
# version_info[item['id']]['etag'] = item['etag']
# version_info[item['id']]['fresh_download'] = not was_versioned
# version_info[item['id']]['time_stamp'] = os.path.getmtime(path) # duh...since we have it!
redis_set(r_c, item, os.path.getmtime(path), box_dir_path=BOX_DIR,
fresh_download=not was_versioned, folder=os.path.dirname(path))
break
except (ConnectionResetError, ConnectionError):
crate_logger.debug(traceback.format_exc())
time.sleep(5)
download_queue.task_done()
else:
download_queue.task_done()
def auth_url():
bottle_app.oauth = setup_oauth(r_c, conf_obj, store_tokens_callback)
secret_keys = r_c.get('secret_keys')
if not secret_keys:
secret_keys = []
else:
secret_keys = json.loads(secret_keys.decode(encoding='utf-8', errors='strict'))
if bottle.request.POST.get('diycrate_secret_key') not in secret_keys:
secret_keys.append(str(bottle.request.POST.get('diycrate_secret_key')))
r_c.set('secret_keys', json.dumps(secret_keys))
return json.dumps([el.decode(encoding='utf-8', errors='strict') if isinstance(el, bytes) else el for el in bottle_app.oauth.authenticate(auth_code=bottle.request.POST.get('code'))])
