def load_directory(self, function_args): base_dir = function_args[0] assert os.path.exists(base_dir) directories = [] files = {} for root, _, files in os.walk(base_dir): try: self.local_metadata._opendb(root, try_ro_on_error=True) file_list = [] ignored_count = 0 updated_count = 0 pending_count = 0 pending_bytes = 0 for a_file in files: should_proc, file_stats = _should_process_file( root, a_file, self.file_filter, self.local_metadata, self.ctx ) if (should_proc, file_stats) == (False, None): # Excluido! ignored_count += 1 else: if should_proc is True: pending_count += 1 pending_bytes += file_stats.stats.st_size elif should_proc is False: updated_count += 1 else: assert False, "Invalid value for should_proc: {}".format(should_proc) directory = { "name": root, "files": files, "files_count": len(files), "file_list": file_list, "ignored_count": ignored_count, "updated_count": updated_count, "pending_count": pending_count, "pending_bytes": pending_bytes, } directories.append(directory) finally: self.local_metadata.close() return {"directories": directories}
def action_upload_directory(_child_conn, directory): """ Uploads a directory. """ logger = logging.getLogger('action_upload_directory[{}]'.format(os.getpid())) try: logger.info("action_upload_directory(directory=%s)", directory) _child_conn.send(PROCESS_STARTED) ctx = Context() ctx.set_include_extensions(('jpg',)) file_filter = FileFilter(ctx) local_metadata = LocalMetadata(ctx) glacier = Glacier(ctx) file_list_to_proc = [] bytes_to_backup = 0 for a_file in os.listdir(directory): if not os.path.isfile(os.path.join(directory, a_file)): continue should_proc, file_stats = _should_process_file( directory, a_file, file_filter, local_metadata, ctx) if should_proc: logger.info("INCLUDING %s/%s", directory, a_file) file_list_to_proc.append((a_file, file_stats)) bytes_to_backup += file_stats.stats.st_size else: logger.info("EXCLUDING %s/%s", directory, a_file) bytes_uploaded = 0 start_time = time.time() msg_template = "Uploading file {}/{} - ({} of {}) - ({} kb uploaded / {} kb pending)" + \ " - Speed: {} kb/sec" try: num = 0 for a_file, file_stats in file_list_to_proc: num += 1 if _child_conn.poll(): received = _child_conn.recv() if received == 'STOP': _child_conn.send(PROCESS_CANCELLED) return else: logger.warn("Ignoring received text '{}'".format(received)) bytes_pending_upload = bytes_to_backup - bytes_uploaded bytes_per_sec = int(bytes_uploaded / (time.time() - start_time)) _child_conn.send(msg_template.format(directory, a_file, num, len(file_list_to_proc), (bytes_uploaded / 1024), (bytes_pending_upload / 1024), (bytes_per_sec / 1024))) logger.info("Starting upload of %s/%s", directory, a_file) if DRY_RUN: time.sleep(2) else: glacier_data = glacier.upload_file(directory, a_file) logger.info("Finished upload of %s/%s", directory, a_file) bytes_uploaded += file_stats.stats.st_size if DRY_RUN: pass else: local_metadata.update_metadata(directory, a_file, file_stats, glacier_data) _child_conn.send(PROCESS_FINISH_OK) except: _child_conn.send(PROCESS_FINISH_WITH_ERROR) except: logger.exception("Exception detected") raise