def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# populate
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f:
f.write('testfile3 body')
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [ elt for elt in persistence.persist(self.fs,
traverser,
None,
tdir.path,
sq,
blocksize=20) ]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [ self.backend.get(fname) for fname in files ]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(fname, hash.make_hasher('sha512')(self.backend.get(fname))[1])
def persist(src_path, dst_uri, config):
mpath, label, dpath = dst_uri.split(",")
fs = filesystem.LocalFileSystem()
traverser = traversal.traverse(fs, src_path)
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
mf = list(persistence.persist(fs, traverser, None, src_path, sq, blocksize=2000))
manifest.write_manifest(get_backend_factory(mpath)(), label, mf)
def persist(src_path, dst_uri, config):
mpath, label, dpath = dst_uri.split(',')
fs = filesystem.LocalFileSystem()
traverser = traversal.traverse(fs, src_path)
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
mf = list(
persistence.persist(fs, traverser, None, src_path, sq, blocksize=2000))
manifest.write_manifest(get_backend_factory(mpath)(), label, mf)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# Populate a tree.
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f:
f.write('testfile3 body')
# Traverse it and persist to store.
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [ elt for elt in persistence.persist(self.fs,
traverser,
None,
tdir.path,
sq,
blocksize=20) ]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [ self.backend.get(fname) for fname in files ]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(fname, hash.make_hasher('sha512')(self.backend.get(fname))[1])
# Create another tempdir and materialize into it, and compare
# the results.
with self.fs.tempdir() as rdir:
materialization.materialize(self.fs, rdir.path, manifest, sq)
def rec(refdir, tstdir):
reflst = sorted(self.fs.listdir(refdir))
tstlst = sorted(self.fs.listdir(tstdir))
self.assertEqual(tstlst, reflst)
# todo: confirm file sizes
# todo: confirm file contents
# todo: confirm file meta
for entry in reflst:
entrypath = os.path.join(refdir, entry)
if not self.fs.is_symlink(entrypath) and self.fs.is_dir(entrypath):
rec(entrypath, os.path.join(tstdir, entry))
rec(tdir.path, rdir.path)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(),
CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# populate
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'),
'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'),
'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(
self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'),
'a') as f:
f.write('testfile3 body')
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [
elt for elt in persistence.persist(
self.fs, traverser, None, tdir.path, sq, blocksize=20)
]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [self.backend.get(fname) for fname in files]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(
fname,
hash.make_hasher('sha512')(self.backend.get(fname))[1])
log.info("checking for previously upped blocks...")
for hash in bf_data().list():
hash = hash.strip()
if len(hash) == 512/4:
alg = 'sha512' # TODO: assume this?
uploaded.append( (alg, hash) )
# run persist
fs = filesystem.LocalFileSystem()
traverser = traversal.traverse(fs, src_path)
sq = storagequeue.StorageQueue(bf_data,
CONCURRENCY)
mf = list(persistence.persist(fs,
traverser,
None,
src_path,
sq,
blocksize=blocksize,
skip_blocks=uploaded))
manifest.write_manifest(b_manifest, label, mf)
def materialize(config, src_uri, dst_path, *files):
if len(files) == 0:
files = None
mpath, label, dpath = src_uri.split(',')
fs = filesystem.LocalFileSystem()
fs.mkdir(dst_path)
mf = list(manifest.read_manifest(get_backend_factory(mpath, config)(),
label))
sq = storagequeue.StorageQueue(get_backend_factory(dpath, config),
CONCURRENCY)