def persist(src_path, dst_uri, config):
mpath, label, dpath = dst_uri.split(",")
fs = filesystem.LocalFileSystem()
traverser = traversal.traverse(fs, src_path)
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
mf = list(persistence.persist(fs, traverser, None, src_path, sq, blocksize=2000))
manifest.write_manifest(get_backend_factory(mpath)(), label, mf)
def test_regular_self(self):
with self.fs.tempdir() as tdir:
with self.fs.open(self.path(tdir.path, 'testfile'), 'w'):
pass
lst = [elt for elt in traversal.traverse(self.fs, tdir.path)]
self.assertEqual(len(lst), 1)
def test_directory_traversal(self):
with self.fs.tempdir() as tdir:
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.mkdir(self.path(tdir.path, 'testdir/subdir'))
lst = [elt for elt in traversal.traverse(self.fs, tdir.path)]
self.assertEqual(len(lst), 2)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# populate
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f:
f.write('testfile3 body')
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [ elt for elt in persistence.persist(self.fs,
traverser,
None,
tdir.path,
sq,
blocksize=20) ]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [ self.backend.get(fname) for fname in files ]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(fname, hash.make_hasher('sha512')(self.backend.get(fname))[1])
def test_regular_self(self):
with self.fs.tempdir() as tdir:
with self.fs.open(self.path(tdir.path, 'testfile'), 'w'):
pass
lst = [ elt for elt in traversal.traverse(self.fs, tdir.path) ]
self.assertEqual(len(lst), 1)
def test_directory_traversal(self):
with self.fs.tempdir() as tdir:
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.mkdir(self.path(tdir.path, 'testdir/subdir'))
lst = [ elt for elt in traversal.traverse(self.fs, tdir.path) ]
self.assertEqual(len(lst), 2)
def test_recursive_symlink(self):
with self.fs.tempdir() as tdir:
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.symlink(self.path(tdir.path, 'testdir'),
self.path(tdir.path, 'testdir/symlink_to_parent'))
lst = [elt for elt in traversal.traverse(self.fs, tdir.path)]
self.assertEqual(len(lst), 2)
def persist(src_path, dst_uri, config):
mpath, label, dpath = dst_uri.split(',')
fs = filesystem.LocalFileSystem()
traverser = traversal.traverse(fs, src_path)
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
mf = list(
persistence.persist(fs, traverser, None, src_path, sq, blocksize=2000))
manifest.write_manifest(get_backend_factory(mpath)(), label, mf)
def test_recursive_symlink(self):
with self.fs.tempdir() as tdir:
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.symlink(self.path(tdir.path, 'testdir'),
self.path(tdir.path, 'testdir/symlink_to_parent'))
lst = [ elt for elt in traversal.traverse(self.fs, tdir.path) ]
self.assertEqual(len(lst), 2)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(), CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# Populate a tree.
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'), 'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'), 'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'), 'a') as f:
f.write('testfile3 body')
# Traverse it and persist to store.
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [ elt for elt in persistence.persist(self.fs,
traverser,
None,
tdir.path,
sq,
blocksize=20) ]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [ self.backend.get(fname) for fname in files ]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(fname, hash.make_hasher('sha512')(self.backend.get(fname))[1])
# Create another tempdir and materialize into it, and compare
# the results.
with self.fs.tempdir() as rdir:
materialization.materialize(self.fs, rdir.path, manifest, sq)
def rec(refdir, tstdir):
reflst = sorted(self.fs.listdir(refdir))
tstlst = sorted(self.fs.listdir(tstdir))
self.assertEqual(tstlst, reflst)
# todo: confirm file sizes
# todo: confirm file contents
# todo: confirm file meta
for entry in reflst:
entrypath = os.path.join(refdir, entry)
if not self.fs.is_symlink(entrypath) and self.fs.is_dir(entrypath):
rec(entrypath, os.path.join(tstdir, entry))
rec(tdir.path, rdir.path)
def test_basic(self):
with storagequeue.StorageQueue(lambda: self.make_backend(),
CONCURRENCY) as sq:
with self.fs.tempdir() as tdir:
# populate
self.fs.mkdir(self.path(tdir.path, 'testdir'))
self.fs.open(self.path(tdir.path, 'testdir/testfile'),
'a').close()
with self.fs.open(self.path(tdir.path, 'testdir/testfile2'),
'a') as f:
f.write('this is the body of testfile2')
self.fs.symlink(
self.path(tdir.path, 'testdir/testfile2'),
self.path(tdir.path, 'testdir/testfile2-symlink'))
with self.fs.open(self.path(tdir.path, 'testdir/testfile3'),
'a') as f:
f.write('testfile3 body')
traverser = traversal.traverse(self.fs, tdir.path)
manifest = [
elt for elt in persistence.persist(
self.fs, traverser, None, tdir.path, sq, blocksize=20)
]
#print meta.to_string() + ' ' + path + ' ' + unicode(hashes)
self.assertEqual(len(manifest), 5)
files = self.backend.list()
self.assertEqual(len(files), 3) # two blocks for 2, one for 3
file_contents = [self.backend.get(fname) for fname in files]
self.assertTrue('testfile3 body' in file_contents)
self.assertTrue('this is the body of ' in file_contents)
self.assertTrue('testfile2' in file_contents)
for fname in files:
self.assertEqual(
fname,
hash.make_hasher('sha512')(self.backend.get(fname))[1])
mfs = get_all_manifests(b_manifest)
if len(mfs) != 0:
mfs = zip(*mfs)[1]
uploaded.extend(get_all_blockhashes(mfs))
if conf.get_option('continue').get_required():
log.info("checking for previously upped blocks...")
for hash in bf_data().list():
hash = hash.strip()
if len(hash) == 512/4:
alg = 'sha512' # TODO: assume this?
uploaded.append( (alg, hash) )
# run persist
fs = filesystem.LocalFileSystem()
traverser = traversal.traverse(fs, src_path)
sq = storagequeue.StorageQueue(bf_data,
CONCURRENCY)
mf = list(persistence.persist(fs,
traverser,
None,
src_path,
sq,
blocksize=blocksize,
skip_blocks=uploaded))
manifest.write_manifest(b_manifest, label, mf)
def materialize(config, src_uri, dst_path, *files):
if len(files) == 0:
files = None
mpath, label, dpath = src_uri.split(',')
def test_basic(self):
with self.fs.tempdir() as tdir:
lst = [elt for elt in traversal.traverse(self.fs, tdir.path)]
self.assertEqual(len(lst), 0)
def test_basic(self):
with self.fs.tempdir() as tdir:
lst = [ elt for elt in traversal.traverse(self.fs, tdir.path) ]
self.assertEqual(len(lst), 0)
