def test_basic(self): with self.make_backend() as b: def make_entry(n, contents): hashes = [] while contents: hashes.append(('sha512', contents[0:5])) # treat contents as hash, nevermind contents = contents[5:] return ('%d' % (n,), md.FileMetaData.from_string('-rwxr-xr-x 5 6 7 8 9 10'), hashes) entries_in = [ make_entry(n, 'contents'.join([ unicode(m) for m in xrange(0, n)])) for n in xrange(0, 100) ] manifest.write_manifest(b, 'test_manifest', entries_in) entries_out = [ entry for entry in manifest.read_manifest(b, 'test_manifest') ] #print b.get('test_manifest') self.assertEqual(manifest.list_manifests(b), [ 'test_manifest' ]) manifest.delete_manifest(b, 'test_manifest') self.assertEqual(manifest.list_manifests(b), []) def to_comparable(entry): path, md, algos = entry return (path, md.to_string(), algos) self.assertEqual([ to_comparable(entry) for entry in entries_in ], [ to_comparable(entry) for entry in entries_out])
def show_manifest(config, uri, label): def number_group(n, sep): # TODO: if boto didn't bork out we would use locale instead of re: # return locale.format('%d', attr.size, grouping=True) return re.sub(r'(\d)(?=(\d{3})+$)', r"\1" + sep, str(n)) b = get_backend_factory(uri, config)() print "%10s %7s %14s %s" % ('Attr','Blocks','Bytes','Name') print "-" * (10 + 7 + 14 + 2 + 10) totblocks = 0 totsize = 0 for name,attr,sums in manifest.read_manifest(b, label): totblocks += len(sums) totsize += attr.size print "%10s %7s %14s %s" % ( str(attr).split(' ',1)[0], number_group(len(sums), "'"), number_group(attr.size, "'"), name, ) print "-" * (10 + 7 + 14 + 2 + 10) print "%10s %7s %14s %s" % ('', number_group(totblocks, "'"), number_group(totsize, "'"), 'Total' )
def list_manifest(uri, config): b = get_backend_factory(uri)() print "%-20s %6s %7s" % ("Manifest", "Files", "Blocks") for mft in b.list(): mf = list(manifest.read_manifest(b, mft)) flatten = lambda z: reduce(lambda x, y: x + y, z) print "%-20s %6d %7d" % (mft, len(mf), len(flatten([x[2] for x in mf])))
def materialize(src_uri, dst_path, config): mpath, label, dpath = src_uri.split(",") fs = filesystem.LocalFileSystem() fs.mkdir(dst_path) mf = list(manifest.read_manifest(get_backend_factory(mpath)(), label)) sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY) materialization.materialize(fs, dst_path, mf, sq)
def materialize(src_uri, dst_path, config): mpath, label, dpath = src_uri.split(',') fs = filesystem.LocalFileSystem() fs.mkdir(dst_path) mf = list(manifest.read_manifest(get_backend_factory(mpath)(), label)) sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY) materialization.materialize(fs, dst_path, mf, sq)
def list_manifest(uri, config): b = get_backend_factory(uri)() print "%-20s %6s %7s" % ('Manifest', 'Files', 'Blocks') for mft in b.list(): mf = list(manifest.read_manifest(b, mft)) flatten = lambda z: reduce(lambda x, y: x + y, z) print "%-20s %6d %7d" % (mft, len(mf), len(flatten([x[2] for x in mf])))
def materialize(config, src_uri, dst_path, *files): if len(files) == 0: files = None mpath, label, dpath = src_uri.split(',') fs = filesystem.LocalFileSystem() fs.mkdir(dst_path) mf = list(manifest.read_manifest(get_backend_factory(mpath, config)(), label)) sq = storagequeue.StorageQueue(get_backend_factory(dpath, config), CONCURRENCY) materialization.materialize(fs, dst_path, mf, sq, files)
def common_blocks(config, uri, *mf_names): b = get_backend_factory(uri, config)() mfs = [manifest.read_manifest(b, x) for x in mf_names] blocks = [get_all_blockhashes([x]) for x in mfs] all_blocks = flatten(blocks) before = [len(x) for x in blocks] [ [bl.remove(x) for bl in blocks] for x in list(set(all_blocks)) if all_blocks.count(x) == len(mf_names) ] after = [len(x) for x in blocks] for nm,bf,af in zip(mf_names,before,after): print '%d unique in %s' % (af, nm) print '%d in common' % (before[0] - after[0])
def test_basic(self): with self.make_backend() as b: def make_entry(n, contents): hashes = [] while contents: hashes.append( ('sha512', contents[0:5])) # treat contents as hash, nevermind contents = contents[5:] return ('%d' % (n, ), md.FileMetaData.from_string('-rwxr-xr-x 5 6 7 8 9 10'), hashes) entries_in = [ make_entry(n, 'contents'.join([unicode(m) for m in xrange(0, n)])) for n in xrange(0, 100) ] manifest.write_manifest(b, 'test_manifest', entries_in) entries_out = [ entry for entry in manifest.read_manifest(b, 'test_manifest') ] #print b.get('test_manifest') self.assertEqual(manifest.list_manifests(b), ['test_manifest']) manifest.delete_manifest(b, 'test_manifest') self.assertEqual(manifest.list_manifests(b), []) def to_comparable(entry): path, md, algos = entry return (path, md.to_string(), algos) self.assertEqual([to_comparable(entry) for entry in entries_in], [to_comparable(entry) for entry in entries_out])
def get_all_manifests(be): return [(x, list(manifest.read_manifest(be, x))) for x in manifest.list_manifests(be)]