def materialize(src_uri, dst_path, config):
mpath, label, dpath = src_uri.split(',')
fs = filesystem.LocalFileSystem()
fs.mkdir(dst_path)
mf = list(manifest.read_manifest(get_backend_factory(mpath)(), label))
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
materialization.materialize(fs, dst_path, mf, sq)
def test_bulk_ops(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
COUNT = 100
puts = [ storagequeue.PutOperation(prefix(str(n)), str(n)) for n in xrange(0, COUNT) ]
gets = [ storagequeue.GetOperation(prefix(str(n))) for n in xrange(0, COUNT) ]
dels = [ storagequeue.DeleteOperation(prefix(str(n))) for n in xrange(0, COUNT) ]
for p in puts:
sq.enqueue(p)
sq.barrier()
for g in gets:
sq.enqueue(g)
sq.barrier()
for d in dels:
sq.enqueue(d)
sq.wait()
for op in puts + gets + dels:
self.assertTrue(op.is_done())
self.assertTrue(op.succeeded())
self.assertEqual([g.value() for g in gets], [ str(n) for n in xrange(0, COUNT) ])
def persist(src_path, dst_uri, config):
mpath, label, dpath = dst_uri.split(',')
fs = filesystem.LocalFileSystem()
traverser = traversal.traverse(fs, src_path)
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
mf = list(
persistence.persist(fs, traverser, None, src_path, sq, blocksize=2000))
manifest.write_manifest(get_backend_factory(mpath)(), label, mf)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# Populate a tree.
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f:
f.write('testfile3 body')
# Traverse it and persist to store.
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [ elt for elt in persistence.persist(self.fs,
traverser,
None,
tdir.path,
sq,
blocksize=20) ]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [ self.backend.get(fname) for fname in files ]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(fname, hash.make_hasher('sha512')(self.backend.get(fname))[1])
# Create another tempdir and materialize into it, and compare
# the results.
with self.fs.tempdir() as rdir:
materialization.materialize(self.fs, rdir.path, manifest, sq)
def rec(refdir, tstdir):
reflst = sorted(self.fs.listdir(refdir))
tstlst = sorted(self.fs.listdir(tstdir))
self.assertEqual(tstlst, reflst)
# todo: confirm file sizes
# todo: confirm file contents
# todo: confirm file meta
for entry in reflst:
entrypath = os.path.join(refdir, entry)
if not self.fs.is_symlink(entrypath) and self.fs.is_dir(entrypath):
rec(entrypath, os.path.join(tstdir, entry))
rec(tdir.path, rdir.path)
def test_bad_put_fail(self):
class FailingPut(storagequeue.PutOperation):
def execute(self, backend):
raise AssertionError('put failed for unit testing purposes')
with logging.FakeLogger(storagequeue, 'log'):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
p1 = FailingPut(prefix('test1'), 'data')
sq.enqueue(p1)
self.assertRaises(storagequeue.OperationHasFailed, sq.wait)
def test_bad_get_fail(self):
with logging.FakeLogger(storagequeue, 'log'):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
p1 = storagequeue.PutOperation(prefix('test1'), 'data')
g1 = storagequeue.GetOperation(prefix('test1'))
g2 = storagequeue.GetOperation(prefix('test2'))
sq.enqueue(p1)
sq.enqueue(g1)
sq.enqueue(g2)
self.assertRaises(storagequeue.OperationHasFailed, sq.wait)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
puts = [ storagequeue.PutOperation(prefix('test%s' % (n,)), str(n)) for n in xrange(0, 5) ]
gets = [ storagequeue.GetOperation(prefix('test%s' % (n,))) for n in xrange(0, 5) ]
dels = [ storagequeue.DeleteOperation(prefix('test%s' % (n,))) for n in xrange(0, 5) ]
# PUT
for p in puts:
self.assertFalse(p.is_done())
self.assertRaises(AssertionError, lambda: p.value())
self.assertRaises(AssertionError, lambda: p.succeeded())
sq.enqueue(p)
sq.wait()
for p in puts:
self.assertTrue(p.is_done())
self.assertTrue(p.succeeded())
# GET + DELETE
for g in gets:
self.assertFalse(g.is_done())
self.assertRaises(AssertionError, lambda: g.value())
self.assertRaises(AssertionError, lambda: g.succeeded())
sq.enqueue(g)
sq.barrier()
for d in dels:
self.assertFalse(d.is_done())
self.assertRaises(AssertionError, lambda: d.value())
self.assertRaises(AssertionError, lambda: d.succeeded())
sq.enqueue(d)
for d in dels:
d.wait() # test individual op waits
sq.wait()
for g in gets:
self.assertTrue(p.is_done())
self.assertTrue(p.succeeded())
self.assertEqual([g.value() for g in gets], [ '0', '1', '2', '3', '4' ])
for d in dels:
self.assertTrue(d.is_done())
self.assertTrue(d.succeeded())
self.assertEqual(d.value(), None)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(),
CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# populate
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'),
'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'),
'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(
self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'),
'a') as f:
f.write('testfile3 body')
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [
elt for elt in persistence.persist(
self.fs, traverser, None, tdir.path, sq, blocksize=20)
]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [self.backend.get(fname) for fname in files]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(
fname,
hash.make_hasher('sha512')(self.backend.get(fname))[1])
if len(mfs) != 0:
mfs = zip(*mfs)[1]
uploaded.extend(get_all_blockhashes(mfs))
if conf.get_option('continue').get_required():
log.info("checking for previously upped blocks...")
for hash in bf_data().list():
hash = hash.strip()
if len(hash) == 512/4:
alg = 'sha512' # TODO: assume this?
uploaded.append( (alg, hash) )
# run persist
fs = filesystem.LocalFileSystem()
traverser = traversal.traverse(fs, src_path)
sq = storagequeue.StorageQueue(bf_data,
CONCURRENCY)
mf = list(persistence.persist(fs,
traverser,
None,
src_path,
sq,
blocksize=blocksize,
skip_blocks=uploaded))
manifest.write_manifest(b_manifest, label, mf)
def materialize(config, src_uri, dst_path, *files):
if len(files) == 0:
files = None
mpath, label, dpath = src_uri.split(',')
fs = filesystem.LocalFileSystem()
fs.mkdir(dst_path)