def testArchiver(self): archives = [] def test_cb(uuid, archive_path): archives.append((uuid, archive_path)) self.ar.set_callback(test_cb) for f in glob.glob('{}/*'.format(TEST_FILES_DIR)): uukey, size = uukey_and_size(f) self.ar.add_file(f, uukey, size) self.ar.close() print(archives)
def do_check(conf, old_tree, target_name): errors = [] done_files = 0 skipped_files = 0 total_files = sum(1 for i in files_to_consider(conf, target_name)) for (full_path, target_path) in files_to_consider(conf, target_name): if os.path.islink(full_path): # TODO skipped_files += 1 continue done_files += 1 if done_files % 100 == 0: log(ProgressReport(done_files, total_files)) mtime_dt = datetime.utcfromtimestamp(os.path.getmtime(full_path)) check = True try: old_entry = old_tree.entries[target_path] if old_entry.last_hashed < mtime_dt: # File has been modified, can't check consistency check = False except KeyError: check = False if not check: skipped_files += 1 continue uukey, file_size = uukey_and_size(full_path) if uukey != old_entry.uuid: print("WARNING: hash mismatch! {}".format(target_path)) errors.append(target_path) print() print("Finished consistency check:") print("{} files on disk ({} skipped), {} in tree".format(total_files, skipped_files, len(old_tree.entries))) print() if len(errors) > 0: print("{} errors:".format(len(errors))) for e in errors: print(e) else: print("No errors.")
def do_freeze(conf, old_tree, target_name): if not conf.has_option('targets', target_name): print("ERROR: target {} doesn't exist".format(target_name)) return None dry_run = conf.getboolean('options', 'dry-run') new_tree = old_tree.copy() print("new_tree: {}".format(new_tree.entries)) uploader = FileUploader(conf, conf.st) uploader.start() def store_file_small(full_path, uukey, target_path): uuid = None if not dry_run: uuid = ar.add_file(full_path, uukey, file_size) new_tree.file_pack[uukey] = uuid def store_file_large(full_path, uukey, target_path): if not dry_run: # Uploads to Glacier in another thread uploader.store(full_path, uukey) new_tree.uuid_to_storage[uukey] = None def store_archive(ar_uuid, arpath): if not dry_run: uploader.store(arpath, ar_uuid) new_tree.uuid_to_storage[ar_uuid] = None def store_symlink(full_path, target_path): symlink_target = os.path.realpath(full_path) if symlink_target.startswith(root_path): symlink_target = os.path.relpath(symlink_target, os.path.dirname(full_path)) new_tree.entries[target_path] = tree.SymlinkEntry(symlink_target) root_path = conf.get('targets', target_name) ar = archiver.Archiver(conf, target_name) ar.set_callback(store_archive) for (full_path, target_path) in files_to_consider(conf, target_name): log(StartedProcessingFile(target_path, full_path)) try: sb = os.stat(full_path) except OSError as e: if e.errno == errno.ENOENT or e.errno == errno.EPERM: log(ProcessFileResult('Skip', 'Errno {}'.format(e.errno))) continue # Should we skip this file? skip_reason = should_skip(conf, sb, target_path, old_tree) if skip_reason: log(ProcessFileResult('Skip', skip_reason)) continue # Symlinks if os.path.islink(full_path): store_symlink(full_path, target_path) log(ProcessFileResult('Symlink')) continue # Hash and check if data already stored uukey, file_size = uukey_and_size(full_path) if conf.getboolean('options', 'tree-only'): new_tree.files[target_path] = tree.TreeEntry(uukey, None) log(ProcessFileResult('{}'.format(uukey[:32]))) continue # TODO: posix stuff new_tree.entries[target_path] = \ tree.FileEntry(sb.st_uid, sb.st_gid, stat.S_IMODE(sb.st_mode), uukey, datetime.utcnow()) if not new_tree.is_stored(uukey): if file_size <= conf.getint('options', 'filesize-limit'): store_file_small(full_path, uukey, target_path) else: store_file_large(full_path, uukey, target_path) log(ProcessFileResult('{}'.format(uukey[:32]))) else: log(ProcessFileResult('Skip', 'Already stored')) # Update archive IDs ar.finish_archive() uploader.to_store.put(UPLOAD_DONE) # Progress indicator while True: num_processed = uploader.progress.get() log(ProgressReport(num_processed, uploader.num_requested)) if num_processed == uploader.num_requested: break uploader.join() # Resolve archive IDs try: for i in range(uploader.num_requested): (uuid, storageid) = uploader.done.get(False) new_tree.uuid_to_storage[uuid] = storageid except queue.Empty: print("ERROR: not enough processed files") if not uploader.done.empty(): print("ERROR: some files unaccounted!") # Check to see that all UUIDs in tree have storage tag for uuid, stag in new_tree.uuid_to_storage.items(): if stag is None: print("ERROR: uuid {} doesn't have storage tag".format(uuid)) # Store the new tree return new_tree