def load_directory(self, function_args):
base_dir = function_args[0]
assert os.path.exists(base_dir)
directories = []
files = {}
for root, _, files in os.walk(base_dir):
try:
self.local_metadata._opendb(root, try_ro_on_error=True)
file_list = []
ignored_count = 0
updated_count = 0
pending_count = 0
pending_bytes = 0
for a_file in files:
should_proc, file_stats = _should_process_file(
root, a_file, self.file_filter, self.local_metadata, self.ctx
)
if (should_proc, file_stats) == (False, None):
# Excluido!
ignored_count += 1
else:
if should_proc is True:
pending_count += 1
pending_bytes += file_stats.stats.st_size
elif should_proc is False:
updated_count += 1
else:
assert False, "Invalid value for should_proc: {}".format(should_proc)
directory = {
"name": root,
"files": files,
"files_count": len(files),
"file_list": file_list,
"ignored_count": ignored_count,
"updated_count": updated_count,
"pending_count": pending_count,
"pending_bytes": pending_bytes,
}
directories.append(directory)
finally:
self.local_metadata.close()
return {"directories": directories}
def action_upload_directory(_child_conn, directory):
"""
Uploads a directory.
"""
logger = logging.getLogger('action_upload_directory[{}]'.format(os.getpid()))
try:
logger.info("action_upload_directory(directory=%s)", directory)
_child_conn.send(PROCESS_STARTED)
ctx = Context()
ctx.set_include_extensions(('jpg',))
file_filter = FileFilter(ctx)
local_metadata = LocalMetadata(ctx)
glacier = Glacier(ctx)
file_list_to_proc = []
bytes_to_backup = 0
for a_file in os.listdir(directory):
if not os.path.isfile(os.path.join(directory, a_file)):
continue
should_proc, file_stats = _should_process_file(
directory, a_file, file_filter, local_metadata, ctx)
if should_proc:
logger.info("INCLUDING %s/%s", directory, a_file)
file_list_to_proc.append((a_file, file_stats))
bytes_to_backup += file_stats.stats.st_size
else:
logger.info("EXCLUDING %s/%s", directory, a_file)
bytes_uploaded = 0
start_time = time.time()
msg_template = "Uploading file {}/{} - ({} of {}) - ({} kb uploaded / {} kb pending)" + \
" - Speed: {} kb/sec"
try:
num = 0
for a_file, file_stats in file_list_to_proc:
num += 1
if _child_conn.poll():
received = _child_conn.recv()
if received == 'STOP':
_child_conn.send(PROCESS_CANCELLED)
return
else:
logger.warn("Ignoring received text '{}'".format(received))
bytes_pending_upload = bytes_to_backup - bytes_uploaded
bytes_per_sec = int(bytes_uploaded / (time.time() - start_time))
_child_conn.send(msg_template.format(directory, a_file,
num, len(file_list_to_proc), (bytes_uploaded / 1024),
(bytes_pending_upload / 1024), (bytes_per_sec / 1024)))
logger.info("Starting upload of %s/%s", directory, a_file)
if DRY_RUN:
time.sleep(2)
else:
glacier_data = glacier.upload_file(directory, a_file)
logger.info("Finished upload of %s/%s", directory, a_file)
bytes_uploaded += file_stats.stats.st_size
if DRY_RUN:
pass
else:
local_metadata.update_metadata(directory, a_file, file_stats, glacier_data)
_child_conn.send(PROCESS_FINISH_OK)
except:
_child_conn.send(PROCESS_FINISH_WITH_ERROR)
except:
logger.exception("Exception detected")
raise
