def persist(src_path, dst_uri, config): mpath, label, dpath = dst_uri.split(",") fs = filesystem.LocalFileSystem() traverser = traversal.traverse(fs, src_path) sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY) mf = list(persistence.persist(fs, traverser, None, src_path, sq, blocksize=2000)) manifest.write_manifest(get_backend_factory(mpath)(), label, mf)
def test_basic(self): with self.make_backend() as b: def make_entry(n, contents): hashes = [] while contents: hashes.append(('sha512', contents[0:5])) # treat contents as hash, nevermind contents = contents[5:] return ('%d' % (n,), md.FileMetaData.from_string('-rwxr-xr-x 5 6 7 8 9 10'), hashes) entries_in = [ make_entry(n, 'contents'.join([ unicode(m) for m in xrange(0, n)])) for n in xrange(0, 100) ] manifest.write_manifest(b, 'test_manifest', entries_in) entries_out = [ entry for entry in manifest.read_manifest(b, 'test_manifest') ] #print b.get('test_manifest') self.assertEqual(manifest.list_manifests(b), [ 'test_manifest' ]) manifest.delete_manifest(b, 'test_manifest') self.assertEqual(manifest.list_manifests(b), []) def to_comparable(entry): path, md, algos = entry return (path, md.to_string(), algos) self.assertEqual([ to_comparable(entry) for entry in entries_in ], [ to_comparable(entry) for entry in entries_out])
def persist(src_path, dst_uri, config): mpath, label, dpath = dst_uri.split(',') fs = filesystem.LocalFileSystem() traverser = traversal.traverse(fs, src_path) sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY) mf = list( persistence.persist(fs, traverser, None, src_path, sq, blocksize=2000)) manifest.write_manifest(get_backend_factory(mpath)(), label, mf)
def test_basic(self): with self.make_backend() as b: def make_entry(n, contents): hashes = [] while contents: hashes.append( ('sha512', contents[0:5])) # treat contents as hash, nevermind contents = contents[5:] return ('%d' % (n, ), md.FileMetaData.from_string('-rwxr-xr-x 5 6 7 8 9 10'), hashes) entries_in = [ make_entry(n, 'contents'.join([unicode(m) for m in xrange(0, n)])) for n in xrange(0, 100) ] manifest.write_manifest(b, 'test_manifest', entries_in) entries_out = [ entry for entry in manifest.read_manifest(b, 'test_manifest') ] #print b.get('test_manifest') self.assertEqual(manifest.list_manifests(b), ['test_manifest']) manifest.delete_manifest(b, 'test_manifest') self.assertEqual(manifest.list_manifests(b), []) def to_comparable(entry): path, md, algos = entry return (path, md.to_string(), algos) self.assertEqual([to_comparable(entry) for entry in entries_in], [to_comparable(entry) for entry in entries_out])
alg = 'sha512' # TODO: assume this? uploaded.append( (alg, hash) ) # run persist fs = filesystem.LocalFileSystem() traverser = traversal.traverse(fs, src_path) sq = storagequeue.StorageQueue(bf_data, CONCURRENCY) mf = list(persistence.persist(fs, traverser, None, src_path, sq, blocksize=blocksize, skip_blocks=uploaded)) manifest.write_manifest(b_manifest, label, mf) def materialize(config, src_uri, dst_path, *files): if len(files) == 0: files = None mpath, label, dpath = src_uri.split(',') fs = filesystem.LocalFileSystem() fs.mkdir(dst_path) mf = list(manifest.read_manifest(get_backend_factory(mpath, config)(), label)) sq = storagequeue.StorageQueue(get_backend_factory(dpath, config), CONCURRENCY) materialization.materialize(fs, dst_path, mf, sq, files) def get_backend_factory(uri, config): """get_backend_factory(uri, config)