def testArchiver(self):
archives = []
def test_cb(uuid, archive_path):
archives.append((uuid, archive_path))
self.ar.set_callback(test_cb)
for f in glob.glob('{}/*'.format(TEST_FILES_DIR)):
uukey, size = uukey_and_size(f)
self.ar.add_file(f, uukey, size)
self.ar.close()
print(archives)
def do_check(conf, old_tree, target_name):
errors = []
done_files = 0
skipped_files = 0
total_files = sum(1 for i in files_to_consider(conf, target_name))
for (full_path, target_path) in files_to_consider(conf, target_name):
if os.path.islink(full_path):
# TODO
skipped_files += 1
continue
done_files += 1
if done_files % 100 == 0:
log(ProgressReport(done_files, total_files))
mtime_dt = datetime.utcfromtimestamp(os.path.getmtime(full_path))
check = True
try:
old_entry = old_tree.entries[target_path]
if old_entry.last_hashed < mtime_dt:
# File has been modified, can't check consistency
check = False
except KeyError:
check = False
if not check:
skipped_files += 1
continue
uukey, file_size = uukey_and_size(full_path)
if uukey != old_entry.uuid:
print("WARNING: hash mismatch! {}".format(target_path))
errors.append(target_path)
print()
print("Finished consistency check:")
print("{} files on disk ({} skipped), {} in tree".format(total_files, skipped_files, len(old_tree.entries)))
print()
if len(errors) > 0:
print("{} errors:".format(len(errors)))
for e in errors:
print(e)
else:
print("No errors.")
def do_freeze(conf, old_tree, target_name):
if not conf.has_option('targets', target_name):
print("ERROR: target {} doesn't exist".format(target_name))
return None
dry_run = conf.getboolean('options', 'dry-run')
new_tree = old_tree.copy()
print("new_tree: {}".format(new_tree.entries))
uploader = FileUploader(conf, conf.st)
uploader.start()
def store_file_small(full_path, uukey, target_path):
uuid = None
if not dry_run:
uuid = ar.add_file(full_path, uukey, file_size)
new_tree.file_pack[uukey] = uuid
def store_file_large(full_path, uukey, target_path):
if not dry_run:
# Uploads to Glacier in another thread
uploader.store(full_path, uukey)
new_tree.uuid_to_storage[uukey] = None
def store_archive(ar_uuid, arpath):
if not dry_run:
uploader.store(arpath, ar_uuid)
new_tree.uuid_to_storage[ar_uuid] = None
def store_symlink(full_path, target_path):
symlink_target = os.path.realpath(full_path)
if symlink_target.startswith(root_path):
symlink_target = os.path.relpath(symlink_target, os.path.dirname(full_path))
new_tree.entries[target_path] = tree.SymlinkEntry(symlink_target)
root_path = conf.get('targets', target_name)
ar = archiver.Archiver(conf, target_name)
ar.set_callback(store_archive)
for (full_path, target_path) in files_to_consider(conf, target_name):
log(StartedProcessingFile(target_path, full_path))
try:
sb = os.stat(full_path)
except OSError as e:
if e.errno == errno.ENOENT or e.errno == errno.EPERM:
log(ProcessFileResult('Skip', 'Errno {}'.format(e.errno)))
continue
# Should we skip this file?
skip_reason = should_skip(conf, sb, target_path, old_tree)
if skip_reason:
log(ProcessFileResult('Skip', skip_reason))
continue
# Symlinks
if os.path.islink(full_path):
store_symlink(full_path, target_path)
log(ProcessFileResult('Symlink'))
continue
# Hash and check if data already stored
uukey, file_size = uukey_and_size(full_path)
if conf.getboolean('options', 'tree-only'):
new_tree.files[target_path] = tree.TreeEntry(uukey, None)
log(ProcessFileResult('{}'.format(uukey[:32])))
continue
# TODO: posix stuff
new_tree.entries[target_path] = \
tree.FileEntry(sb.st_uid, sb.st_gid, stat.S_IMODE(sb.st_mode),
uukey, datetime.utcnow())
if not new_tree.is_stored(uukey):
if file_size <= conf.getint('options', 'filesize-limit'):
store_file_small(full_path, uukey, target_path)
else:
store_file_large(full_path, uukey, target_path)
log(ProcessFileResult('{}'.format(uukey[:32])))
else:
log(ProcessFileResult('Skip', 'Already stored'))
# Update archive IDs
ar.finish_archive()
uploader.to_store.put(UPLOAD_DONE)
# Progress indicator
while True:
num_processed = uploader.progress.get()
log(ProgressReport(num_processed, uploader.num_requested))
if num_processed == uploader.num_requested:
break
uploader.join()
# Resolve archive IDs
try:
for i in range(uploader.num_requested):
(uuid, storageid) = uploader.done.get(False)
new_tree.uuid_to_storage[uuid] = storageid
except queue.Empty:
print("ERROR: not enough processed files")
if not uploader.done.empty():
print("ERROR: some files unaccounted!")
# Check to see that all UUIDs in tree have storage tag
for uuid, stag in new_tree.uuid_to_storage.items():
if stag is None:
print("ERROR: uuid {} doesn't have storage tag".format(uuid))
# Store the new tree
return new_tree