def setup_rados(args):
global RADOS
RADOS = Rados(conffile='')
RADOS.connect()
try:
RADOS.create_pool(args.pool)
except ObjectExists:
pass
global IOCTX_CAS
global IOCTX_INDEX
IOCTX_CAS = RADOS.open_ioctx(args.pool)
IOCTX_INDEX = RADOS.open_ioctx(args.pool)
global CAS
global CHUNKER
if "compression" in args:
CAS = cas.CAS(IOCTX_CAS, compression=args.compression)
else:
CAS = cas.CAS(IOCTX_CAS)
if "chunk_size" in args:
CHUNKER = Chunker(CAS, IOCTX_INDEX, chunk_size=args.chunk_size)
else:
CHUNKER = Chunker(CAS, IOCTX_INDEX)
def test_chunker_put_get_single():
"""
Test: read(write(x)) = x for x filling only a single chunk
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO(random_bytes(42))
obj_name = random_id()
version = chunker.write_full(obj_name, data_in)
data_out = StringIO()
chunker.read_full(obj_name, data_out, version)
eq_buffer(data_in.getvalue(), data_out.getvalue())
def test_chunker_put_get_single():
"""
Test: read(write(x)) = x for x filling only a single chunk
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO(random_bytes(42))
obj_name = random_id()
version = chunker.write_full(obj_name, data_in)
data_out = StringIO()
chunker.read_full(obj_name, data_out, version)
eq_buffer(data_in.getvalue(), data_out.getvalue())
def test_chunker_versions():
"""
Test: versions / head_version returns version of last write_full. Single write_full
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO(random_bytes(10*1024**1))
obj_name = random_id()
version = chunker.write_full(obj_name, data_in)
eq(len(chunker.versions(obj_name)), 1)
eq(version, chunker.head_version(obj_name))
eq(version, chunker.versions(obj_name)[0])
def test_chunker_partial_read_past_size():
"""
Test: partial reads past *file* size
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO("\x00" * chunker.chunk_size)
obj_name = random_id()
version = chunker.write_full(obj_name, data_in)
buf = chunker.read(obj_name, chunker.chunk_size, chunker.chunk_size, version)
eq_buffer(buf, "")
def test_chunker_put_get_multiple_fraction():
"""
Test: read(write(x)) = x for x spread over multiple chunks. With partially filled chunks
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO(random_bytes(int(chunker.chunk_size * 1.5)))
obj_name = random_id()
version = chunker.write_full(obj_name, data_in)
data_out = StringIO()
chunker.read_full(obj_name, data_out, version)
eq_buffer(data_in.getvalue(), data_out.getvalue())
def test_chunker_put_get_multiple_fraction():
"""
Test: read(write(x)) = x for x spread over multiple chunks. With partially filled chunks
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO(random_bytes(int(chunker.chunk_size*1.5)))
obj_name = random_id()
version = chunker.write_full(obj_name, data_in)
data_out = StringIO()
chunker.read_full(obj_name, data_out, version)
eq_buffer(data_in.getvalue(), data_out.getvalue())
def test_chunker_partial_read_past_size():
"""
Test: partial reads past *file* size
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO("\x00" * chunker.chunk_size)
obj_name = random_id()
version = chunker.write_full(obj_name, data_in)
buf = chunker.read(obj_name, chunker.chunk_size, chunker.chunk_size,
version)
eq_buffer(buf, "")
def test_chunker_partial_read():
"""
Test: partial reads using chunker.read with different input and weird extents
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO("\x00" * chunker.chunk_size +
"\xFF" * chunker.chunk_size)
obj_name = random_id()
version = chunker.write_full(obj_name, data_in)
middle = chunker.chunk_size / 2
buf = chunker.read(obj_name, chunker.chunk_size, middle, version)
eq(len(buf), chunker.chunk_size)
eq_buffer("\x00" * (chunker.chunk_size/2) +
"\xFF" * (chunker.chunk_size/2), buf)
def test_chunker_partial_read():
"""
Test: partial reads using chunker.read with different input and weird extents
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO("\x00" * chunker.chunk_size +
"\xFF" * chunker.chunk_size)
obj_name = random_id()
version = chunker.write_full(obj_name, data_in)
middle = chunker.chunk_size / 2
buf = chunker.read(obj_name, chunker.chunk_size, middle, version)
eq(len(buf), chunker.chunk_size)
eq_buffer(
"\x00" * (chunker.chunk_size / 2) + "\xFF" * (chunker.chunk_size / 2),
buf)
def test_chunker_no_litter():
"""
Test: Write and immediate remove should not leave any object behind
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO(random_bytes(chunker.chunk_size*4))
obj_name = random_id()
chunker.write_full(obj_name, data_in)
chunker.remove_all_versions(obj_name)
cas_objs = [x.key for x in ioctx_cas.list_objects()]
index_objs = [x.key for x in ioctx_index.list_objects()]
print "CAS objects left:", cas_objs
print "Index objects left:", index_objs
eq(len(cas_objs), 0)
eq(len(index_objs), 0)
def test_chunker_remove():
"""
Test: remove actually removes
- `remove_version(write_full)`: No versions, but index object
- `write_full, write_full, remove_all_versions`: Index object gone
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = random_bytes(42)
obj_name = random_id()
version = chunker.write_full(obj_name, StringIO(data_in))
chunker.remove_version(obj_name, version)
eq(chunker.head_version(obj_name), None)
chunker.write_full(obj_name, StringIO(data_in))
chunker.write_full(obj_name, StringIO(data_in))
chunker.remove_all_versions(obj_name)
assert_raises(ObjectNotFound, chunker.head_version, obj_name)
def test_chunker_remove():
"""
Test: remove actually removes
- `remove_version(write_full)`: No versions, but index object
- `write_full, write_full, remove_all_versions`: Index object gone
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = random_bytes(42)
obj_name = random_id()
version = chunker.write_full(obj_name, StringIO(data_in))
chunker.remove_version(obj_name, version)
eq(chunker.head_version(obj_name), None)
chunker.write_full(obj_name, StringIO(data_in))
chunker.write_full(obj_name, StringIO(data_in))
chunker.remove_all_versions(obj_name)
assert_raises(ObjectNotFound, chunker.head_version, obj_name)
def test_chunker_multiple_versions():
"""
Test: versions / head_version return version of last write_full. Multiple write_full
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = random_bytes(42)
obj_name = random_id()
versions = (
chunker.write_full(obj_name, StringIO(data_in)),
chunker.write_full(obj_name, StringIO(data_in)),
chunker.write_full(obj_name, StringIO(data_in)),
chunker.write_full(obj_name, StringIO(data_in)),
chunker.write_full(obj_name, StringIO(data_in)),
)
eq(len(versions), len(chunker.versions(obj_name)))
eq(versions[-1], chunker.head_version(obj_name))
eq(versions[0], chunker.versions(obj_name)[0])
def test_chunker_versions():
"""
Test: versions / head_version returns version of last write_full. Single write_full
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO(random_bytes(10 * 1024**1))
obj_name = random_id()
version = chunker.write_full(obj_name, data_in)
eq(len(chunker.versions(obj_name)), 1)
eq(version, chunker.head_version(obj_name))
eq(version, chunker.versions(obj_name)[0])
def test_chunker_multiple_versions():
"""
Test: versions / head_version return version of last write_full. Multiple write_full
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = random_bytes(42)
obj_name = random_id()
versions = (
chunker.write_full(obj_name, StringIO(data_in)),
chunker.write_full(obj_name, StringIO(data_in)),
chunker.write_full(obj_name, StringIO(data_in)),
chunker.write_full(obj_name, StringIO(data_in)),
chunker.write_full(obj_name, StringIO(data_in)),
)
eq(len(versions), len(chunker.versions(obj_name)))
eq(versions[-1], chunker.head_version(obj_name))
eq(versions[0], chunker.versions(obj_name)[0])
def test_chunker_no_litter():
"""
Test: Write and immediate remove should not leave any object behind
"""
cas = CAS(ioctx_cas)
chunker = Chunker(cas, ioctx_index)
data_in = StringIO(random_bytes(chunker.chunk_size * 4))
obj_name = random_id()
chunker.write_full(obj_name, data_in)
chunker.remove_all_versions(obj_name)
cas_objs = [x.key for x in ioctx_cas.list_objects()]
index_objs = [x.key for x in ioctx_index.list_objects()]
print "CAS objects left:", cas_objs
print "Index objects left:", index_objs
eq(len(cas_objs), 0)
eq(len(index_objs), 0)