def test_blob_walk(metasync, opts):
"test creating/walking a blob dir"
opts.notree = True
bs = blobs.BlobStore2(metasync)
root = blobs.BlobDir2(bs)
# generate sample tree
for i in range(1, 3):
parent_dir = blobs.BlobDir2(bs)
root.add("dir-%s" % i, parent_dir)
for j in range(1, 4):
child_dir = blobs.BlobDir2(bs)
parent_dir.add("sub-%s" % j, child_dir)
for k in range(1, 5):
blob_file = blobs.BlobFile2(bs)
child_dir.add("file-%s" % k, blob_file)
# count all entries
cnt = 0
for (name, blob) in root.walk():
dbg.dbg("%-18s: %s" % (name, blob.hv))
cnt += 1
assert cnt == 2 * 3 * 4 + 2 * 3 + 2
def test_blob_file(metasync, opts):
"test blobfile-related operations"
test_init(metasync, opts)
bs = blobs.BlobStore2(metasync)
blob_file = blobs.BlobFile2(bs)
# empty file
assert blob_file.hv is not None \
and len(blob_file.entries) == 0
# random file with 3 chunks (last one is smaller than unit)
unit = 1 * MB
size = 3 * MB - 2 * KB
pn = os.path.join(opts.tmpdir, "file-%s" % size)
util.create_random_file(pn, size)
# store each chunk to blob_file
blob_file = bs.load_file(pn, unit)
# check entries and total size
assert len(blob_file.entries) == 3 and blob_file.size == size
# test store/load
blob_file.store()
# loaded from disk
loaded_blob = blobs.BlobFile2(bs, blob_file.hv)
assert loaded_blob.dump() == blob_file.dump()
def test_large_blob(metasync, opts):
test_init(metasync, opts)
bs = blobs.BlobStore2(metasync)
blob_dir = blobs.MBlobDir2(bs)
pn = os.path.join(opts.root, "a")
with open(pn, "w") as f:
f.write("hello world")
blob_dir.add_file("a", pn)
pn = os.path.join(opts.root, "b")
with open(pn, "w") as f:
f.write("hello world2")
pn = os.path.join(opts.root, "b")
blob_dir.add_file("b", pn)
blob_dir.store()
def test_blob_diff(metasync, opts):
"test blob diff operation"
test_init(metasync, opts)
bs = blobs.BlobStore2(metasync)
blob_dir = blobs.BlobDir2(bs)
blob_dir.add("file", blobs.BlobFile2(bs))
blob_dir_sub = blobs.BlobDir2(bs)
blob_dir.add("dir1", blob_dir_sub)
blob_dir_sub.add("file2", blobs.BlobFile2(bs))
blob_dir2 = blobs.BlobDir2(bs)
blob_dir2.add("file", blobs.BlobFile2(bs))
blob_dir.diff(blob_dir2)
def test_blob(metasync, opts):
"test blob-related operations"
test_init(metasync, opts)
bs = blobs.BlobStore2(metasync)
blob_dir = blobs.BlobDir2(bs)
# empty dir
assert blob_dir.hv is not None \
and len(blob_dir.entries) == 0
# add three
hv0 = blob_dir.hv
blob_dir.add("dir1", blobs.BlobDir2(bs))
blob_dir.add("dir2", blobs.BlobDir2(bs))
blob_dir.add("dir3", blobs.BlobDir2(bs))
blob_dir.add("file", blobs.BlobFile2(bs))
hv3 = blob_dir.hv
assert hv0 != hv3 \
and len(blob_dir.entries) == 4
for (name, blob) in blob_dir.entries.iteritems():
# empty dir
if isinstance(blob, blobs.BlobDir2):
assert blob.hv == hv0
# empty file
if isinstance(blob, blobs.BlobFile2):
assert blob.hv != hv0
# delete one
blob_dir.rm("dir2")
hv2 = blob_dir.hv
assert hv3 != hv2 \
and len(blob_dir.entries) == 3
dbg.dbg("hv: %s\n%s" % (hv2, blob_dir.dump()))
# test store/load
blob_dir.store()
# loaded from disk
loaded_blob = blobs.BlobDir2(bs, hv2)
assert loaded_blob.dump() == blob_dir.dump()
def test_blob_load(metasync, opts):
"test loading file/dir from a path"
_init_disk_metasync(metasync, opts)
bs = blobs.BlobStore2(metasync)
# /a/b/c
dirp = metasync.get_local_path("a", "b", "c")
util.mkdirs(dirp)
# /a/b/c/file
pn = os.path.join(dirp, "file")
util.create_random_file(pn, 5 * KB)
blob = bs.load_dir(dirp)
blob.add("file", bs.load_file(pn))
# count how many blobs
root = bs.get_root_blob()
dbg.dbg("%-15s: %s" % ("/", root.hv))
cnt = 0
for (name, blob) in bs.walk():
dbg.dbg("%-15s: %s" % (name, blob.hv))
cnt += 1
assert cnt == len(["a", "b", "c", "file"])
# flush all new blobs
assert len(os.listdir(metasync.path_objs)) == 0
root.store()
assert len(os.listdir(metasync.path_objs)) == 6
# "." => root
test_blob = bs.load_dir(metasync.get_local_path("."))
assert test_blob == root
test_blob = bs.load_dir(metasync.get_local_path(""))
assert test_blob == root