def test_basic(self):
with self.make_backend() as b:
def make_entry(n, contents):
hashes = []
while contents:
hashes.append(('sha512', contents[0:5])) # treat contents as hash, nevermind
contents = contents[5:]
return ('%d' % (n,), md.FileMetaData.from_string('-rwxr-xr-x 5 6 7 8 9 10'), hashes)
entries_in = [ make_entry(n, 'contents'.join([ unicode(m) for m in xrange(0, n)])) for n in xrange(0, 100) ]
manifest.write_manifest(b, 'test_manifest', entries_in)
entries_out = [ entry for entry in manifest.read_manifest(b, 'test_manifest') ]
#print b.get('test_manifest')
self.assertEqual(manifest.list_manifests(b), [ 'test_manifest' ])
manifest.delete_manifest(b, 'test_manifest')
self.assertEqual(manifest.list_manifests(b), [])
def to_comparable(entry):
path, md, algos = entry
return (path, md.to_string(), algos)
self.assertEqual([ to_comparable(entry) for entry in entries_in ],
[ to_comparable(entry) for entry in entries_out])
def show_manifest(config, uri, label):
def number_group(n, sep):
# TODO: if boto didn't bork out we would use locale instead of re:
# return locale.format('%d', attr.size, grouping=True)
return re.sub(r'(\d)(?=(\d{3})+$)',
r"\1" + sep,
str(n))
b = get_backend_factory(uri, config)()
print "%10s %7s %14s %s" % ('Attr','Blocks','Bytes','Name')
print "-" * (10 + 7 + 14 + 2 + 10)
totblocks = 0
totsize = 0
for name,attr,sums in manifest.read_manifest(b, label):
totblocks += len(sums)
totsize += attr.size
print "%10s %7s %14s %s" % (
str(attr).split(' ',1)[0],
number_group(len(sums), "'"),
number_group(attr.size, "'"),
name,
)
print "-" * (10 + 7 + 14 + 2 + 10)
print "%10s %7s %14s %s" % ('',
number_group(totblocks, "'"),
number_group(totsize, "'"),
'Total'
)
def list_manifest(uri, config):
b = get_backend_factory(uri)()
print "%-20s %6s %7s" % ("Manifest", "Files", "Blocks")
for mft in b.list():
mf = list(manifest.read_manifest(b, mft))
flatten = lambda z: reduce(lambda x, y: x + y, z)
print "%-20s %6d %7d" % (mft, len(mf), len(flatten([x[2] for x in mf])))
def materialize(src_uri, dst_path, config):
mpath, label, dpath = src_uri.split(",")
fs = filesystem.LocalFileSystem()
fs.mkdir(dst_path)
mf = list(manifest.read_manifest(get_backend_factory(mpath)(), label))
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
materialization.materialize(fs, dst_path, mf, sq)
def materialize(src_uri, dst_path, config):
mpath, label, dpath = src_uri.split(',')
fs = filesystem.LocalFileSystem()
fs.mkdir(dst_path)
mf = list(manifest.read_manifest(get_backend_factory(mpath)(), label))
sq = storagequeue.StorageQueue(get_backend_factory(dpath), CONCURRENCY)
materialization.materialize(fs, dst_path, mf, sq)
def list_manifest(uri, config):
b = get_backend_factory(uri)()
print "%-20s %6s %7s" % ('Manifest', 'Files', 'Blocks')
for mft in b.list():
mf = list(manifest.read_manifest(b, mft))
flatten = lambda z: reduce(lambda x, y: x + y, z)
print "%-20s %6d %7d" % (mft, len(mf), len(flatten([x[2]
for x in mf])))
def materialize(config, src_uri, dst_path, *files):
if len(files) == 0:
files = None
mpath, label, dpath = src_uri.split(',')
fs = filesystem.LocalFileSystem()
fs.mkdir(dst_path)
mf = list(manifest.read_manifest(get_backend_factory(mpath, config)(),
label))
sq = storagequeue.StorageQueue(get_backend_factory(dpath, config),
CONCURRENCY)
materialization.materialize(fs, dst_path, mf, sq, files)
def common_blocks(config, uri, *mf_names):
b = get_backend_factory(uri, config)()
mfs = [manifest.read_manifest(b, x) for x in mf_names]
blocks = [get_all_blockhashes([x]) for x in mfs]
all_blocks = flatten(blocks)
before = [len(x) for x in blocks]
[ [bl.remove(x) for bl in blocks]
for x in list(set(all_blocks))
if all_blocks.count(x) == len(mf_names)
]
after = [len(x) for x in blocks]
for nm,bf,af in zip(mf_names,before,after):
print '%d unique in %s' % (af, nm)
print '%d in common' % (before[0] - after[0])
def test_basic(self):
with self.make_backend() as b:
def make_entry(n, contents):
hashes = []
while contents:
hashes.append(
('sha512',
contents[0:5])) # treat contents as hash, nevermind
contents = contents[5:]
return ('%d' % (n, ),
md.FileMetaData.from_string('-rwxr-xr-x 5 6 7 8 9 10'),
hashes)
entries_in = [
make_entry(n,
'contents'.join([unicode(m) for m in xrange(0, n)]))
for n in xrange(0, 100)
]
manifest.write_manifest(b, 'test_manifest', entries_in)
entries_out = [
entry for entry in manifest.read_manifest(b, 'test_manifest')
]
#print b.get('test_manifest')
self.assertEqual(manifest.list_manifests(b), ['test_manifest'])
manifest.delete_manifest(b, 'test_manifest')
self.assertEqual(manifest.list_manifests(b), [])
def to_comparable(entry):
path, md, algos = entry
return (path, md.to_string(), algos)
self.assertEqual([to_comparable(entry) for entry in entries_in],
[to_comparable(entry) for entry in entries_out])
def get_all_manifests(be):
return [(x, list(manifest.read_manifest(be, x)))
for x in manifest.list_manifests(be)]