def materialize(src_uri, dst_path, config):
mpath, label, dpath = src_uri.split(",")
fs = filesystem.LocalFileSystem()
fs.mkdir(dst_path)
mf = list(manifest.read_manifest(get_backend_factory(mpath)(), label))
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
materialization.materialize(fs, dst_path, mf, sq)
def materialize(src_uri, dst_path, config):
mpath, label, dpath = src_uri.split(',')
fs = filesystem.LocalFileSystem()
fs.mkdir(dst_path)
mf = list(manifest.read_manifest(get_backend_factory(mpath)(), label))
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
materialization.materialize(fs, dst_path, mf, sq)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# Populate a tree.
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f:
f.write('testfile3 body')
# Traverse it and persist to store.
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [ elt for elt in persistence.persist(self.fs,
traverser,
None,
tdir.path,
sq,
blocksize=20) ]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [ self.backend.get(fname) for fname in files ]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(fname, hash.make_hasher('sha512')(self.backend.get(fname))[1])
# Create another tempdir and materialize into it, and compare
# the results.
with self.fs.tempdir() as rdir:
materialization.materialize(self.fs, rdir.path, manifest, sq)
def rec(refdir, tstdir):
reflst = sorted(self.fs.listdir(refdir))
tstlst = sorted(self.fs.listdir(tstdir))
self.assertEqual(tstlst, reflst)
# todo: confirm file sizes
# todo: confirm file contents
# todo: confirm file meta
for entry in reflst:
entrypath = os.path.join(refdir, entry)
if not self.fs.is_symlink(entrypath) and self.fs.is_dir(entrypath):
rec(entrypath, os.path.join(tstdir, entry))
rec(tdir.path, rdir.path)
def materialize(config, src_uri, dst_path, *files):
if len(files) == 0:
files = None
mpath, label, dpath = src_uri.split(',')
fs = filesystem.LocalFileSystem()
fs.mkdir(dst_path)
mf = list(manifest.read_manifest(get_backend_factory(mpath, config)(),
label))
sq = storagequeue.StorageQueue(get_backend_factory(dpath, config),
CONCURRENCY)
materialization.materialize(fs, dst_path, mf, sq, files)
