def download_queue_processor(): """ Implements a simple re-try mechanism for pending downloads :return: """ while True: if download_queue.not_empty: item, path, oauth = download_queue.get() # blocks if item['type'] == 'file': info = redis_get(r_c, item) if r_c.exists(redis_key(item['id'])) else None # client = Client(oauth) # keep it around for easy access # hack because we did not use to store the file_path, but do not want to force a download if info and 'file_path' not in info: info['file_path'] = path r_c.set(redis_key(item['id']), json.dumps(info)) r_c.set('diy_crate.last_save_time_stamp', int(time.time())) # no version, or diff version, or the file does not exist locally if not info or info['etag'] != item['etag'] or not os.path.exists(path): try: for i in range(15): if os.path.basename(path).startswith('.~lock'): # avoid downloading lock files break try: with open(path, 'wb') as item_handler: crate_logger.debug('About to download: {obj_name}, ' '{obj_id}'.format(obj_name=item['name'], obj_id=item['id'])) item.download_to(item_handler) path_to_add = os.path.dirname(path) wm.add_watch(path=path_to_add, mask=mask, rec=True, auto_add=True) notify_user_with_gui('Downloaded: {}'.format(path)) except BoxAPIException as e: crate_logger.debug(traceback.format_exc()) if e.status == 404: crate_logger.debug('Apparently item: {obj_id}, {path} has been deleted, ' 'right before we tried to download'.format(obj_id=item['id'], path=path)) break was_versioned = r_c.exists(redis_key(item['id'])) # # version_info[item['id']] = version_info.get(item['id'], {'etag': item['etag'], # 'fresh_download': True, # 'time_stamp': time.time()}) # version_info[item['id']]['etag'] = item['etag'] # version_info[item['id']]['fresh_download'] = not was_versioned # version_info[item['id']]['time_stamp'] = os.path.getmtime(path) # duh...since we have it! redis_set(r_c, item, os.path.getmtime(path), box_dir_path=BOX_DIR, fresh_download=not was_versioned, folder=os.path.dirname(path)) break except (ConnectionResetError, ConnectionError): crate_logger.debug(traceback.format_exc()) time.sleep(5) download_queue.task_done() else: download_queue.task_done()
def auth_url(): bottle_app.oauth = setup_oauth(r_c, conf_obj, store_tokens_callback) secret_keys = r_c.get('secret_keys') if not secret_keys: secret_keys = [] else: secret_keys = json.loads(secret_keys.decode(encoding='utf-8', errors='strict')) if bottle.request.POST.get('diycrate_secret_key') not in secret_keys: secret_keys.append(str(bottle.request.POST.get('diycrate_secret_key'))) r_c.set('secret_keys', json.dumps(secret_keys)) return json.dumps([el.decode(encoding='utf-8', errors='strict') if isinstance(el, bytes) else el for el in bottle_app.oauth.authenticate(auth_code=bottle.request.POST.get('code'))])