def test_store(self):
def dfs_file_checksums(path):
for file_name in os.listdir(path):
file_path = os.path.join(path, file_name)
assert os.path.isdir(file_path) or os.path.isfile(file_path)
if os.path.isdir(file_path):
yield from dfs_file_checksums(file_path)
elif os.path.islink(file_path):
message = xxhash.xxh64()
message.update(os.readlink(file_path))
yield message.hexdigest()
elif os.path.isfile(file_path):
BLOCKSIZE = 65536
message = xxhash.xxh64()
with open(file_path, 'rb') as f:
file_buffer = f.read(BLOCKSIZE)
while len(file_buffer) > 0:
message.update(file_buffer)
file_buffer = f.read(BLOCKSIZE)
yield message.hexdigest()
checkpoint = Checkpoint.build_checkpoint(self.TEST_RESOURCES_PATH)
with tempfile.TemporaryDirectory() as temp_path:
storage = FileSystemStorage(temp_path)
storage.store(self.TEST_RESOURCES_PATH, checkpoint)
remote_file_checksums = os.listdir(
os.path.join(temp_path, FileSystemStorage.FILE_DIR))
file_checksums = list(dfs_file_checksums(self.TEST_RESOURCES_PATH))
self.assertCountEqual(remote_file_checksums, file_checksums)
def create_fs(checkpoint_name=None, path=None):
if path is None:
path = get_fs_path()
src_path = os.getcwd()
checkpoint = Checkpoint.build_checkpoint(src_path, checkpoint_name)
storage = FileSystemStorage(path)
storage.store(src_path, checkpoint)
def list_fs(path=None):
if path is None:
path = get_fs_path()
storage = FileSystemStorage(path)
checkpoint_metas = storage.retrieve_checkpoint_metas()
if not checkpoint_metas:
click.echo('No checkpoints found.')
else:
for checkpoint_meta in checkpoint_metas:
echo_checkpoint_meta(checkpoint_meta)
def test_checkpoint_file_without_name(self):
checkpoint = Checkpoint.build_checkpoint(self.resources_path)
with tempfile.TemporaryDirectory() as tmp_path:
storage = FileSystemStorage(tmp_path)
storage.store(self.resources_path, checkpoint)
meta = storage.retrieve_checkpoint_metas()[0]
self.assertEqual(meta.checksum, checkpoint.root.checksum)
self.assertEqual(meta.time, checkpoint.time)
self.assertIs(meta.name, None)
def test_checkpoint_retrieval(self):
checkpoint = Checkpoint.build_checkpoint(self.resources_path,
'my_test_name')
with tempfile.TemporaryDirectory() as tmp_path:
storage = FileSystemStorage(tmp_path)
storage.store(self.resources_path, checkpoint)
meta = storage.retrieve_checkpoint_metas()[0]
checkpoint_2 = storage.retrieve_checkpoint(meta)
self.assertEqual(checkpoint.time, checkpoint_2.time)
self.assertEqual(checkpoint.name, checkpoint_2.name)
self.assertEqual(checkpoint.meta.checksum, checkpoint_2.meta.checksum)
self.assertEqual(checkpoint.root.checksum, checkpoint_2.root.checksum)
def restore_fs(identifier, path=None):
if path is None:
path = get_fs_path()
dst_path = os.getcwd()
storage = FileSystemStorage(path)
try:
storage.retrieve_by_checksum(dst_path, identifier)
except FileNotFoundError:
try:
storage.retrieve_by_name(dst_path, identifier)
except FileNotFoundError:
click.echo('No checkpoints matching identifier: ' + identifier)
except NoUniqueMatchError:
click.echo('Multiple checkpoints matching identifier: ' +
identifier)
except NoUniqueMatchError:
click.echo('Multiple checkpoints matching identifier: ' + identifier)
from bakker.checkpoint import Checkpoint
from bakker.storage import FileSystemStorage
import sys
import time
in_path = sys.argv[1]
out_path = sys.argv[2]
start = time.time()
checkpoint = Checkpoint.build_checkpoint(in_path)
checkpoint_end = time.time()
storage = FileSystemStorage(out_path)
storage.store(in_path, checkpoint)
end = time.time()
print("Checkpoint build duration:" + str(checkpoint_end - start))
print("Total duration: " + str(end - start))
def test_retrieve(self):
tmp_store_dir = tempfile.TemporaryDirectory()
tmp_store_path = tmp_store_dir.name
tmp_retrieve_dir_a = tempfile.TemporaryDirectory()
tmp_retrieve_dir_b = tempfile.TemporaryDirectory()
tmp_retrieve_dir_c = tempfile.TemporaryDirectory()
tmp_retrieve_path_a = tmp_retrieve_dir_a.name
tmp_retrieve_path_b = tmp_retrieve_dir_b.name
tmp_retrieve_path_c = tmp_retrieve_dir_c.name
storage = FileSystemStorage(tmp_store_path)
checkpoint = Checkpoint.build_checkpoint(self.TEST_RESOURCES_PATH,
'checkpoint_name')
storage.store(self.TEST_RESOURCES_PATH, checkpoint)
checkpoint_meta = storage.retrieve_checkpoint_metas()[0]
storage.retrieve(tmp_retrieve_path_a, checkpoint_meta)
storage.retrieve_by_checksum(tmp_retrieve_path_b,
checkpoint_meta.checksum[:5])
storage.retrieve_by_name(tmp_retrieve_path_c, checkpoint_meta.name)
dir_comparison_a = filecmp.dircmp(self.TEST_RESOURCES_PATH,
tmp_retrieve_path_a)
dir_comparison_b = filecmp.dircmp(self.TEST_RESOURCES_PATH,
tmp_retrieve_path_b)
dir_comparison_c = filecmp.dircmp(self.TEST_RESOURCES_PATH,
tmp_retrieve_path_c)
self.assertEqual(dir_comparison_a.diff_files, [])
self.assertEqual(dir_comparison_b.diff_files, [])
self.assertEqual(dir_comparison_c.diff_files, [])
for dir_path, dir_names, filenames in os.walk(self.TEST_RESOURCES_PATH,
followlinks=False):
rel_path = os.path.relpath(dir_path, self.TEST_RESOURCES_PATH)
for f in filenames:
original_path = os.path.join(self.TEST_RESOURCES_PATH,
rel_path, f)
retrieve_path = os.path.join(tmp_retrieve_path_a, rel_path, f)
file_permissions_origin = stat.S_IMODE(
os.lstat(original_path).st_mode)
file_permissions_restored = stat.S_IMODE(
os.lstat(retrieve_path).st_mode)
self.assertEqual(file_permissions_origin,
file_permissions_restored)
# remove tmp directories after test
tmp_store_dir.cleanup()
tmp_retrieve_dir_a.cleanup()
tmp_retrieve_dir_b.cleanup()
tmp_retrieve_dir_c.cleanup()