def download_queue_processor(): """ Implements a simple re-try mechanism for pending downloads :return: """ while True: if download_queue.not_empty: item, path, oauth = download_queue.get() # blocks if item['type'] == 'file': info = redis_get(r_c, item) if r_c.exists(redis_key(item['id'])) else None # client = Client(oauth) # keep it around for easy access # hack because we did not use to store the file_path, but do not want to force a download if info and 'file_path' not in info: info['file_path'] = path r_c.set(redis_key(item['id']), json.dumps(info)) r_c.set('diy_crate.last_save_time_stamp', int(time.time())) # no version, or diff version, or the file does not exist locally if not info or info['etag'] != item['etag'] or not os.path.exists(path): try: for i in range(15): if os.path.basename(path).startswith('.~lock'): # avoid downloading lock files break try: with open(path, 'wb') as item_handler: crate_logger.debug('About to download: {obj_name}, ' '{obj_id}'.format(obj_name=item['name'], obj_id=item['id'])) item.download_to(item_handler) path_to_add = os.path.dirname(path) wm.add_watch(path=path_to_add, mask=mask, rec=True, auto_add=True) notify_user_with_gui('Downloaded: {}'.format(path)) except BoxAPIException as e: crate_logger.debug(traceback.format_exc()) if e.status == 404: crate_logger.debug('Apparently item: {obj_id}, {path} has been deleted, ' 'right before we tried to download'.format(obj_id=item['id'], path=path)) break was_versioned = r_c.exists(redis_key(item['id'])) # # version_info[item['id']] = version_info.get(item['id'], {'etag': item['etag'], # 'fresh_download': True, # 'time_stamp': time.time()}) # version_info[item['id']]['etag'] = item['etag'] # version_info[item['id']]['fresh_download'] = not was_versioned # version_info[item['id']]['time_stamp'] = os.path.getmtime(path) # duh...since we have it! redis_set(r_c, item, os.path.getmtime(path), box_dir_path=BOX_DIR, fresh_download=not was_versioned, folder=os.path.dirname(path)) break except (ConnectionResetError, ConnectionError): crate_logger.debug(traceback.format_exc()) time.sleep(5) download_queue.task_done() else: download_queue.task_done()
def walk_and_notify_and_download_tree(path, box_folder, client, oauth_obj, p_id=None): """ Walk the path recursively and add watcher and create the path. :param path: :param box_folder: :param client: :param oauth_obj: :param p_id: :return: """ if os.path.isdir(path): wm.add_watch(path, mask, rec=True, auto_add=True) local_files = os.listdir(path) b_folder = client.folder(folder_id=box_folder['id']).get() num_entries_in_folder = b_folder['item_collection']['total_count'] limit = 100 for offset in range(0, num_entries_in_folder, limit): for box_item in b_folder.get_items(limit=limit, offset=offset): if box_item['name'] in local_files: local_files.remove(box_item['name']) for local_file in local_files: # prioritize the local_files not yet on box's server. cur_box_folder = b_folder local_path = os.path.join(path, local_file) if os.path.isfile(local_path): upload_queue.put([os.path.getmtime(local_path), partial(cur_box_folder.upload, local_path, local_file), oauth_obj]) ids_in_folder = [] for offset in range(0, num_entries_in_folder, limit): for box_item in b_folder.get_items(limit=limit, offset=offset): ids_in_folder.append(box_item['id']) if box_item['name'] in local_files: local_files.remove(box_item['name']) if box_item['type'] == 'folder': local_path = os.path.join(path, box_item['name']) fresh_download = False if not os.path.isdir(local_path): os.mkdir(local_path) fresh_download = True retry_limit = 15 for i in range(0, retry_limit): try: redis_set(cache_client=r_c, cloud_item=box_item, last_modified_time=os.path.getmtime(local_path), box_dir_path=BOX_DIR, fresh_download=fresh_download, folder=os.path.dirname(local_path)) walk_and_notify_and_download_tree(local_path, client.folder(folder_id=box_item['id']).get(), client, oauth_obj, p_id=box_folder['id']) break except BoxAPIException as e: crate_logger.debug(traceback.format_exc()) if e.status == 404: crate_logger.debug('Box says: {obj_id}, ' '{obj_name}, is a 404 status.'.format(obj_id=box_item['id'], obj_name=box_item[ 'name'])) crate_logger.debug( 'But, this is a folder, we do not handle recursive folder deletes correctly yet.') break except (ConnectionError, ConnectionResetError, BrokenPipeError): crate_logger.debug('Attempt {idx}/{limit}; {the_trace}'.format(the_trace=traceback.format_exc(), idx=i+1, limit=retry_limit)) else: try: file_obj = box_item download_queue.put((file_obj, os.path.join(path, box_item['name']), oauth_obj)) except BoxAPIException as e: crate_logger.debug(traceback.format_exc()) if e.status == 404: crate_logger.debug('Box says: {obj_id}, {obj_name}, ' 'is a 404 status.'.format(obj_id=box_item['id'], obj_name=box_item['name'])) if r_c.exists(redis_key(box_item['id'])): crate_logger.debug('Deleting {obj_id}, ' '{obj_name}'.format(obj_id=box_item['id'], obj_name=box_item['name'])) r_c.delete(redis_key(box_item['id'])) redis_set(cache_client=r_c, cloud_item=b_folder, last_modified_time=os.path.getmtime(path), box_dir_path=BOX_DIR, fresh_download=not r_c.exists(redis_key(box_folder['id'])), folder=os.path.dirname(path), sub_ids=ids_in_folder, parent_id=p_id)