def download_queue_processor():
"""
Implements a simple re-try mechanism for pending downloads
:return:
"""
while True:
if download_queue.not_empty:
item, path, oauth = download_queue.get() # blocks
if item['type'] == 'file':
info = redis_get(r_c, item) if r_c.exists(redis_key(item['id'])) else None
# client = Client(oauth) # keep it around for easy access
# hack because we did not use to store the file_path, but do not want to force a download
if info and 'file_path' not in info:
info['file_path'] = path
r_c.set(redis_key(item['id']), json.dumps(info))
r_c.set('diy_crate.last_save_time_stamp', int(time.time()))
# no version, or diff version, or the file does not exist locally
if not info or info['etag'] != item['etag'] or not os.path.exists(path):
try:
for i in range(15):
if os.path.basename(path).startswith('.~lock'): # avoid downloading lock files
break
try:
with open(path, 'wb') as item_handler:
crate_logger.debug('About to download: {obj_name}, '
'{obj_id}'.format(obj_name=item['name'], obj_id=item['id']))
item.download_to(item_handler)
path_to_add = os.path.dirname(path)
wm.add_watch(path=path_to_add, mask=mask, rec=True, auto_add=True)
notify_user_with_gui('Downloaded: {}'.format(path))
except BoxAPIException as e:
crate_logger.debug(traceback.format_exc())
if e.status == 404:
crate_logger.debug('Apparently item: {obj_id}, {path} has been deleted, '
'right before we tried to download'.format(obj_id=item['id'],
path=path))
break
was_versioned = r_c.exists(redis_key(item['id']))
#
# version_info[item['id']] = version_info.get(item['id'], {'etag': item['etag'],
# 'fresh_download': True,
# 'time_stamp': time.time()})
# version_info[item['id']]['etag'] = item['etag']
# version_info[item['id']]['fresh_download'] = not was_versioned
# version_info[item['id']]['time_stamp'] = os.path.getmtime(path) # duh...since we have it!
redis_set(r_c, item, os.path.getmtime(path), box_dir_path=BOX_DIR,
fresh_download=not was_versioned, folder=os.path.dirname(path))
break
except (ConnectionResetError, ConnectionError):
crate_logger.debug(traceback.format_exc())
time.sleep(5)
download_queue.task_done()
else:
download_queue.task_done()
def upload_queue_processor():
"""
Implements a simple re-try mechanism for pending uploads
:return:
"""
while True:
if upload_queue.not_empty:
callable_up = upload_queue.get() # blocks
# TODO: pass in the actual item being updated/uploaded, so we can do more intelligent retry mechanisms
was_list = isinstance(callable_up, list)
last_modified_time = oauth = None
if was_list:
last_modified_time, callable_up, oauth = callable_up
args = callable_up.args if isinstance(callable_up, partial) else None
num_retries = 15
for x in range(15):
try:
ret_val = callable_up()
if was_list:
item = ret_val # is the new/updated item
if isinstance(item, File):
client = Client(oauth)
file_obj = client.file(file_id=item.object_id).get()
redis_set(r_c, file_obj, last_modified_time, box_dir_path=BOX_DIR)
break
except BoxAPIException as e:
crate_logger.debug('{the_args}, {the_trace}'.format(the_args=args,
the_trace=traceback.format_exc()))
if e.status == 409:
crate_logger.debug('Apparently Box says this item already exists...'
'and we were trying to create it. Need to handle this better')
break
except (ConnectionError, BrokenPipeError, ProtocolError, ConnectionResetError):
time.sleep(3)
crate_logger.debug('{the_args}, {the_trace}'.format(the_args=args,
the_trace=traceback.format_exc()))
if x >= num_retries - 1:
crate_logger.debug('Upload giving up on: {}'.format(callable_up))
# no immediate plans to do anything with this info, yet.
uploads_given_up_on.append(callable_up)
except (TypeError, FileNotFoundError):
crate_logger.debug(traceback.format_exc())
break
upload_queue.task_done()
def walk_and_notify_and_download_tree(path, box_folder, client, oauth_obj, p_id=None):
"""
Walk the path recursively and add watcher and create the path.
:param path:
:param box_folder:
:param client:
:param oauth_obj:
:param p_id:
:return:
"""
if os.path.isdir(path):
wm.add_watch(path, mask, rec=True, auto_add=True)
local_files = os.listdir(path)
b_folder = client.folder(folder_id=box_folder['id']).get()
num_entries_in_folder = b_folder['item_collection']['total_count']
limit = 100
for offset in range(0, num_entries_in_folder, limit):
for box_item in b_folder.get_items(limit=limit, offset=offset):
if box_item['name'] in local_files:
local_files.remove(box_item['name'])
for local_file in local_files: # prioritize the local_files not yet on box's server.
cur_box_folder = b_folder
local_path = os.path.join(path, local_file)
if os.path.isfile(local_path):
upload_queue.put([os.path.getmtime(local_path), partial(cur_box_folder.upload, local_path, local_file),
oauth_obj])
ids_in_folder = []
for offset in range(0, num_entries_in_folder, limit):
for box_item in b_folder.get_items(limit=limit, offset=offset):
ids_in_folder.append(box_item['id'])
if box_item['name'] in local_files:
local_files.remove(box_item['name'])
if box_item['type'] == 'folder':
local_path = os.path.join(path, box_item['name'])
fresh_download = False
if not os.path.isdir(local_path):
os.mkdir(local_path)
fresh_download = True
retry_limit = 15
for i in range(0, retry_limit):
try:
redis_set(cache_client=r_c, cloud_item=box_item,
last_modified_time=os.path.getmtime(local_path),
box_dir_path=BOX_DIR, fresh_download=fresh_download,
folder=os.path.dirname(local_path))
walk_and_notify_and_download_tree(local_path,
client.folder(folder_id=box_item['id']).get(),
client, oauth_obj,
p_id=box_folder['id'])
break
except BoxAPIException as e:
crate_logger.debug(traceback.format_exc())
if e.status == 404:
crate_logger.debug('Box says: {obj_id}, '
'{obj_name}, is a 404 status.'.format(obj_id=box_item['id'],
obj_name=box_item[
'name']))
crate_logger.debug(
'But, this is a folder, we do not handle recursive folder deletes correctly yet.')
break
except (ConnectionError, ConnectionResetError, BrokenPipeError):
crate_logger.debug('Attempt {idx}/{limit}; {the_trace}'.format(the_trace=traceback.format_exc(),
idx=i+1, limit=retry_limit))
else:
try:
file_obj = box_item
download_queue.put((file_obj, os.path.join(path, box_item['name']), oauth_obj))
except BoxAPIException as e:
crate_logger.debug(traceback.format_exc())
if e.status == 404:
crate_logger.debug('Box says: {obj_id}, {obj_name}, '
'is a 404 status.'.format(obj_id=box_item['id'], obj_name=box_item['name']))
if r_c.exists(redis_key(box_item['id'])):
crate_logger.debug('Deleting {obj_id}, '
'{obj_name}'.format(obj_id=box_item['id'], obj_name=box_item['name']))
r_c.delete(redis_key(box_item['id']))
redis_set(cache_client=r_c, cloud_item=b_folder, last_modified_time=os.path.getmtime(path),
box_dir_path=BOX_DIR, fresh_download=not r_c.exists(redis_key(box_folder['id'])),
folder=os.path.dirname(path),
sub_ids=ids_in_folder, parent_id=p_id)
