def setup_rados(args): global RADOS RADOS = Rados(conffile='') RADOS.connect() try: RADOS.create_pool(args.pool) except ObjectExists: pass global IOCTX_CAS global IOCTX_INDEX IOCTX_CAS = RADOS.open_ioctx(args.pool) IOCTX_INDEX = RADOS.open_ioctx(args.pool) global CAS global CHUNKER if "compression" in args: CAS = cas.CAS(IOCTX_CAS, compression=args.compression) else: CAS = cas.CAS(IOCTX_CAS) if "chunk_size" in args: CHUNKER = Chunker(CAS, IOCTX_INDEX, chunk_size=args.chunk_size) else: CHUNKER = Chunker(CAS, IOCTX_INDEX)
def test_chunker_put_get_single(): """ Test: read(write(x)) = x for x filling only a single chunk """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = StringIO(random_bytes(42)) obj_name = random_id() version = chunker.write_full(obj_name, data_in) data_out = StringIO() chunker.read_full(obj_name, data_out, version) eq_buffer(data_in.getvalue(), data_out.getvalue())
def test_chunker_versions(): """ Test: versions / head_version returns version of last write_full. Single write_full """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = StringIO(random_bytes(10*1024**1)) obj_name = random_id() version = chunker.write_full(obj_name, data_in) eq(len(chunker.versions(obj_name)), 1) eq(version, chunker.head_version(obj_name)) eq(version, chunker.versions(obj_name)[0])
def test_chunker_partial_read_past_size(): """ Test: partial reads past *file* size """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = StringIO("\x00" * chunker.chunk_size) obj_name = random_id() version = chunker.write_full(obj_name, data_in) buf = chunker.read(obj_name, chunker.chunk_size, chunker.chunk_size, version) eq_buffer(buf, "")
def test_chunker_put_get_multiple_fraction(): """ Test: read(write(x)) = x for x spread over multiple chunks. With partially filled chunks """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = StringIO(random_bytes(int(chunker.chunk_size * 1.5))) obj_name = random_id() version = chunker.write_full(obj_name, data_in) data_out = StringIO() chunker.read_full(obj_name, data_out, version) eq_buffer(data_in.getvalue(), data_out.getvalue())
def test_chunker_put_get_multiple_fraction(): """ Test: read(write(x)) = x for x spread over multiple chunks. With partially filled chunks """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = StringIO(random_bytes(int(chunker.chunk_size*1.5))) obj_name = random_id() version = chunker.write_full(obj_name, data_in) data_out = StringIO() chunker.read_full(obj_name, data_out, version) eq_buffer(data_in.getvalue(), data_out.getvalue())
def test_chunker_partial_read(): """ Test: partial reads using chunker.read with different input and weird extents """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = StringIO("\x00" * chunker.chunk_size + "\xFF" * chunker.chunk_size) obj_name = random_id() version = chunker.write_full(obj_name, data_in) middle = chunker.chunk_size / 2 buf = chunker.read(obj_name, chunker.chunk_size, middle, version) eq(len(buf), chunker.chunk_size) eq_buffer("\x00" * (chunker.chunk_size/2) + "\xFF" * (chunker.chunk_size/2), buf)
def test_chunker_partial_read(): """ Test: partial reads using chunker.read with different input and weird extents """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = StringIO("\x00" * chunker.chunk_size + "\xFF" * chunker.chunk_size) obj_name = random_id() version = chunker.write_full(obj_name, data_in) middle = chunker.chunk_size / 2 buf = chunker.read(obj_name, chunker.chunk_size, middle, version) eq(len(buf), chunker.chunk_size) eq_buffer( "\x00" * (chunker.chunk_size / 2) + "\xFF" * (chunker.chunk_size / 2), buf)
def test_chunker_no_litter(): """ Test: Write and immediate remove should not leave any object behind """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = StringIO(random_bytes(chunker.chunk_size*4)) obj_name = random_id() chunker.write_full(obj_name, data_in) chunker.remove_all_versions(obj_name) cas_objs = [x.key for x in ioctx_cas.list_objects()] index_objs = [x.key for x in ioctx_index.list_objects()] print "CAS objects left:", cas_objs print "Index objects left:", index_objs eq(len(cas_objs), 0) eq(len(index_objs), 0)
def test_chunker_remove(): """ Test: remove actually removes - `remove_version(write_full)`: No versions, but index object - `write_full, write_full, remove_all_versions`: Index object gone """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = random_bytes(42) obj_name = random_id() version = chunker.write_full(obj_name, StringIO(data_in)) chunker.remove_version(obj_name, version) eq(chunker.head_version(obj_name), None) chunker.write_full(obj_name, StringIO(data_in)) chunker.write_full(obj_name, StringIO(data_in)) chunker.remove_all_versions(obj_name) assert_raises(ObjectNotFound, chunker.head_version, obj_name)
def test_chunker_multiple_versions(): """ Test: versions / head_version return version of last write_full. Multiple write_full """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = random_bytes(42) obj_name = random_id() versions = ( chunker.write_full(obj_name, StringIO(data_in)), chunker.write_full(obj_name, StringIO(data_in)), chunker.write_full(obj_name, StringIO(data_in)), chunker.write_full(obj_name, StringIO(data_in)), chunker.write_full(obj_name, StringIO(data_in)), ) eq(len(versions), len(chunker.versions(obj_name))) eq(versions[-1], chunker.head_version(obj_name)) eq(versions[0], chunker.versions(obj_name)[0])
def test_chunker_versions(): """ Test: versions / head_version returns version of last write_full. Single write_full """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = StringIO(random_bytes(10 * 1024**1)) obj_name = random_id() version = chunker.write_full(obj_name, data_in) eq(len(chunker.versions(obj_name)), 1) eq(version, chunker.head_version(obj_name)) eq(version, chunker.versions(obj_name)[0])
def test_chunker_no_litter(): """ Test: Write and immediate remove should not leave any object behind """ cas = CAS(ioctx_cas) chunker = Chunker(cas, ioctx_index) data_in = StringIO(random_bytes(chunker.chunk_size * 4)) obj_name = random_id() chunker.write_full(obj_name, data_in) chunker.remove_all_versions(obj_name) cas_objs = [x.key for x in ioctx_cas.list_objects()] index_objs = [x.key for x in ioctx_index.list_objects()] print "CAS objects left:", cas_objs print "Index objects left:", index_objs eq(len(cas_objs), 0) eq(len(index_objs), 0)