Beispiel #1
1
 def setUp(self):
     global rados2
     rados2 = Rados(conffile='')
     rados2.connect()
     global ioctx2
     ioctx2 = rados2.open_ioctx(pool_name)
     create_image()
Beispiel #2
0
def setup_rados(args):
    global RADOS
    RADOS = Rados(conffile='')
    RADOS.connect()

    try:
        RADOS.create_pool(args.pool)
    except ObjectExists:
        pass

    global IOCTX_CAS
    global IOCTX_INDEX
    IOCTX_CAS = RADOS.open_ioctx(args.pool)
    IOCTX_INDEX = RADOS.open_ioctx(args.pool)

    global CAS
    global CHUNKER

    if "compression" in args:
        CAS = cas.CAS(IOCTX_CAS, compression=args.compression)
    else:
        CAS = cas.CAS(IOCTX_CAS)

    if "chunk_size" in args:
        CHUNKER = Chunker(CAS, IOCTX_INDEX, chunk_size=args.chunk_size)
    else:
        CHUNKER = Chunker(CAS, IOCTX_INDEX)
Beispiel #3
0
def setup_rados(args):
    global RADOS
    RADOS = Rados(conffile='')
    RADOS.connect()

    try:
        RADOS.create_pool(args.pool)
    except ObjectExists:
        pass

    global IOCTX_CAS
    global IOCTX_INDEX
    IOCTX_CAS = RADOS.open_ioctx(args.pool)
    IOCTX_INDEX = RADOS.open_ioctx(args.pool)

    global CAS
    global CHUNKER

    if "compression" in args:
        CAS = cas.CAS(IOCTX_CAS, compression=args.compression)
    else:
        CAS = cas.CAS(IOCTX_CAS)

    if "chunk_size" in args:
        CHUNKER = Chunker(CAS, IOCTX_INDEX, chunk_size=args.chunk_size)
    else:
        CHUNKER = Chunker(CAS, IOCTX_INDEX)
Beispiel #4
0
def setup_module():
    global rados
    rados = Rados(conffile='')
    rados.connect()

    global pool_name
    pool_name = random_id()
    rados.create_pool(pool_name)

    global ioctx_cas
    global ioctx_index
    ioctx_cas = rados.open_ioctx(pool_name)
    ioctx_index = rados.open_ioctx(pool_name)
Beispiel #5
0
def setup_module():
    global rados
    rados = Rados(conffile='')
    rados.connect()

    global pool_name
    pool_name = random_id()
    rados.create_pool(pool_name)

    global ioctx_cas
    global ioctx_index
    ioctx_cas = rados.open_ioctx(pool_name)
    ioctx_index = rados.open_ioctx(pool_name)
Beispiel #6
0
def setUp():
    global rados
    rados = Rados(conffile='')
    rados.connect()
    assert rados.pool_exists('rbd')
    global ioctx
    ioctx = rados.open_ioctx('rbd')
Beispiel #7
0
class TestIoctxEc(object):

    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.pool = 'test-ec'
        self.profile = 'testprofile-%s' % self.pool
        cmd = {"prefix": "osd erasure-code-profile set", 
               "name": self.profile, "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}
        ret, buf, out = self.rados.mon_command(json.dumps(cmd), b'', timeout=30)
        eq(ret, 0, msg=out)
        # create ec pool with profile created above
        cmd = {'prefix': 'osd pool create', 'pg_num': 8, 'pgp_num': 8,
               'pool': self.pool, 'pool_type': 'erasure', 
               'erasure_code_profile': self.profile}
        ret, buf, out = self.rados.mon_command(json.dumps(cmd), b'', timeout=30)
        eq(ret, 0, msg=out)
        assert self.rados.pool_exists(self.pool)
        self.ioctx = self.rados.open_ioctx(self.pool)

    def tearDown(self):
        cmd = {"prefix": "osd unset", "key": "noup"}
        self.rados.mon_command(json.dumps(cmd), b'')
        self.ioctx.close()
        self.rados.delete_pool(self.pool)
        self.rados.shutdown()

    def test_alignment(self):
        eq(self.ioctx.alignment(), 8192)
Beispiel #8
0
def setUp():
    global rados
    rados = Rados(conffile='')
    rados.connect()
    assert rados.pool_exists('rbd')
    global ioctx
    ioctx = rados.open_ioctx('rbd')
Beispiel #9
0
class TestObject(object):
    def setUp(self):
        self.rados = Rados(conffile="")
        self.rados.connect()
        self.rados.create_pool("test_pool")
        assert self.rados.pool_exists("test_pool")
        self.ioctx = self.rados.open_ioctx("test_pool")
        self.ioctx.write("foo", b"bar")
        self.object = Object(self.ioctx, "foo")

    def tearDown(self):
        self.ioctx.close()
        self.ioctx = None
        self.rados.delete_pool("test_pool")
        self.rados.shutdown()
        self.rados = None

    def test_read(self):
        eq(self.object.read(3), b"bar")
        eq(self.object.read(100), b"")

    def test_seek(self):
        self.object.write(b"blah")
        self.object.seek(0)
        eq(self.object.read(4), b"blah")
        self.object.seek(1)
        eq(self.object.read(3), b"lah")

    def test_write(self):
        self.object.write(b"barbaz")
        self.object.seek(0)
        eq(self.object.read(3), b"bar")
        eq(self.object.read(3), b"baz")
Beispiel #10
0
 def setUp(self):
     global rados2
     rados2 = Rados(conffile='')
     rados2.connect()
     global ioctx2
     ioctx2 = rados2.open_ioctx(pool_name)
     create_image()
Beispiel #11
0
class TestObject(object):

    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')
        self.ioctx.write('foo', b'bar')
        self.object = Object(self.ioctx, 'foo')

    def tearDown(self):
        self.ioctx.close()
        self.ioctx = None
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()
        self.rados = None

    def test_read(self):
        eq(self.object.read(3), b'bar')
        eq(self.object.read(100), b'')

    def test_seek(self):
        self.object.write(b'blah')
        self.object.seek(0)
        eq(self.object.read(4), b'blah')
        self.object.seek(1)
        eq(self.object.read(3), b'lah')

    def test_write(self):
        self.object.write(b'barbaz')
        self.object.seek(0)
        eq(self.object.read(3), b'bar')
        eq(self.object.read(3), b'baz')
Beispiel #12
0
class TestObject(object):
    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')
        self.ioctx.write('foo', b'bar')
        self.object = Object(self.ioctx, 'foo')

    def tearDown(self):
        self.ioctx.close()
        self.ioctx = None
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()
        self.rados = None

    def test_read(self):
        eq(self.object.read(3), b'bar')
        eq(self.object.read(100), b'')

    def test_seek(self):
        self.object.write(b'blah')
        self.object.seek(0)
        eq(self.object.read(4), b'blah')
        self.object.seek(1)
        eq(self.object.read(3), b'lah')

    def test_write(self):
        self.object.write(b'barbaz')
        self.object.seek(0)
        eq(self.object.read(3), b'bar')
        eq(self.object.read(3), b'baz')
Beispiel #13
0
class TestIoctxEc(object):

    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.pool = 'test-ec'
        self.profile = 'testprofile-%s' % self.pool
        cmd = {"prefix": "osd erasure-code-profile set", 
               "name": self.profile, "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}
        ret, buf, out = self.rados.mon_command(json.dumps(cmd), b'', timeout=30)
        eq(ret, 0, msg=out)
        # create ec pool with profile created above
        cmd = {'prefix': 'osd pool create', 'pg_num': 8, 'pgp_num': 8,
               'pool': self.pool, 'pool_type': 'erasure', 
               'erasure_code_profile': self.profile}
        ret, buf, out = self.rados.mon_command(json.dumps(cmd), b'', timeout=30)
        eq(ret, 0, msg=out)
        assert self.rados.pool_exists(self.pool)
        self.ioctx = self.rados.open_ioctx(self.pool)

    def tearDown(self):
        cmd = {"prefix": "osd unset", "key": "noup"}
        self.rados.mon_command(json.dumps(cmd), b'')
        self.ioctx.close()
        self.rados.delete_pool(self.pool)
        self.rados.shutdown()

    def test_alignment(self):
        eq(self.ioctx.alignment(), 8192)
Beispiel #14
0
def setUp():
    global rados
    rados = Rados(conffile='')
    rados.connect()
    assert rados.pool_exists('rbd')
    global ioctx
    ioctx = rados.open_ioctx('rbd')
    global features
    features = os.getenv("RBD_FEATURES")
    if features is not None:
        features = int(features)
Beispiel #15
0
def setup_module():
    global rados
    rados = Rados(conffile="")
    rados.connect()
    assert rados.pool_exists("rbd")
    global ioctx
    ioctx = rados.open_ioctx("rbd")
    global features
    features = os.getenv("RBD_FEATURES")
    if features is not None:
        features = int(features)
Beispiel #16
0
def setUp():
    global rados
    rados = Rados(conffile='')
    rados.connect()
    assert rados.pool_exists('rbd')
    global ioctx
    ioctx = rados.open_ioctx('rbd')
    global features
    features = os.getenv("RBD_FEATURES")
    if features is not None:
        features = int(features)
Beispiel #17
0
def setup_module():
    global rados
    rados = Rados(conffile='')
    rados.connect()
    global pool_name
    pool_name = get_temp_pool_name()
    rados.create_pool(pool_name)
    global ioctx
    ioctx = rados.open_ioctx(pool_name)
    global features
    features = os.getenv("RBD_FEATURES")
    features = int(features) if features is not None else 61
Beispiel #18
0
def setup_module():
    global rados
    rados = Rados(conffile='')
    rados.connect()
    global pool_name
    pool_name = get_temp_pool_name()
    rados.create_pool(pool_name)
    global ioctx
    ioctx = rados.open_ioctx(pool_name)
    global features
    features = os.getenv("RBD_FEATURES")
    features = int(features) if features is not None else 61
Beispiel #19
0
def main():
    if len(sys.argv) != 2 or sys.argv[1] not in ['master', 'slave']:
        print("usage: %s: [master/slave]" % sys.argv[0])
        sys.exit(2)

    rados = Rados(conffile='')
    rados.connect()
    ioctx = rados.open_ioctx(POOL_NAME)
    if sys.argv[1] == 'master':
        master(ioctx)
    else:
        slave(ioctx)
    rados.shutdown()
class Rados:
    ignored_prefixes = ['_shadow', '_multipart']

    def __init__(self, bucket_marker, ceph_conf):
        self.bucket_marker = bucket_marker
        self.cluster = CephRados(conffile=ceph_conf)
        self.cluster.connect()

    def list_objects(self, rados_pool_name, key_regex):
        filter = compile("{}_(?!({}))({})".format(
            self.bucket_marker, "|".join(self.ignored_prefixes),
            args.key_regex))
        with self.cluster.open_ioctx(rados_pool_name) as ioctx:
            for obj in ioctx.list_objects():
                match = filter.match(obj.key)
                if match is None:
                    continue
                file_path = match.group(2)
                yield file_path
Beispiel #21
0
class TestIoCtxSelfManagedSnaps(object):
    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')

    def tearDown(self):
        cmd = {"prefix": "osd unset", "key": "noup"}
        self.rados.mon_command(json.dumps(cmd), b'')
        self.ioctx.close()
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()

    def test(self):
        # cannot mix-and-match pool and self-managed snapshot mode
        self.ioctx.set_self_managed_snap_write([])
        self.ioctx.write('abc', b'abc')
        snap_id_1 = self.ioctx.create_self_managed_snap()
        self.ioctx.set_self_managed_snap_write([snap_id_1])

        self.ioctx.write('abc', b'def')
        snap_id_2 = self.ioctx.create_self_managed_snap()
        self.ioctx.set_self_managed_snap_write([snap_id_1, snap_id_2])

        self.ioctx.write('abc', b'ghi')

        self.ioctx.rollback_self_managed_snap('abc', snap_id_1)
        eq(self.ioctx.read('abc'), b'abc')

        self.ioctx.rollback_self_managed_snap('abc', snap_id_2)
        eq(self.ioctx.read('abc'), b'def')

        self.ioctx.remove_self_managed_snap(snap_id_1)
        self.ioctx.remove_self_managed_snap(snap_id_2)
Beispiel #22
0
class TestIoCtxSelfManagedSnaps(object):
    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')

    def tearDown(self):
        cmd = {"prefix":"osd unset", "key":"noup"}
        self.rados.mon_command(json.dumps(cmd), b'')
        self.ioctx.close()
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()

    def test(self):
        # cannot mix-and-match pool and self-managed snapshot mode
        self.ioctx.set_self_managed_snap_write([])
        self.ioctx.write('abc', b'abc')
        snap_id_1 = self.ioctx.create_self_managed_snap()
        self.ioctx.set_self_managed_snap_write([snap_id_1])

        self.ioctx.write('abc', b'def')
        snap_id_2 = self.ioctx.create_self_managed_snap()
        self.ioctx.set_self_managed_snap_write([snap_id_1, snap_id_2])

        self.ioctx.write('abc', b'ghi')

        self.ioctx.rollback_self_managed_snap('abc', snap_id_1)
        eq(self.ioctx.read('abc'), b'abc')

        self.ioctx.rollback_self_managed_snap('abc', snap_id_2)
        eq(self.ioctx.read('abc'), b'def')

        self.ioctx.remove_self_managed_snap(snap_id_1)
        self.ioctx.remove_self_managed_snap(snap_id_2)
Beispiel #23
0
class TestIoctx(object):
    def setUp(self):
        self.rados = Rados(conffile="")
        self.rados.connect()
        self.rados.create_pool("test_pool")
        assert self.rados.pool_exists("test_pool")
        self.ioctx = self.rados.open_ioctx("test_pool")

    def tearDown(self):
        cmd = {"prefix": "osd unset", "key": "noup"}
        self.rados.mon_command(json.dumps(cmd), "")
        self.ioctx.close()
        self.rados.delete_pool("test_pool")
        self.rados.shutdown()

    def test_change_auid(self):
        self.ioctx.change_auid(ANONYMOUS_AUID)
        self.ioctx.change_auid(ADMIN_AUID)

    def test_write(self):
        self.ioctx.write("abc", "abc")
        eq(self.ioctx.read("abc"), "abc")

    def test_write_full(self):
        self.ioctx.write("abc", "abc")
        eq(self.ioctx.read("abc"), "abc")
        self.ioctx.write_full("abc", "d")
        eq(self.ioctx.read("abc"), "d")

    def test_append(self):
        self.ioctx.write("abc", "a")
        self.ioctx.append("abc", "b")
        self.ioctx.append("abc", "c")
        eq(self.ioctx.read("abc"), "abc")

    def test_write_zeros(self):
        self.ioctx.write("abc", "a\0b\0c")
        eq(self.ioctx.read("abc"), "a\0b\0c")

    def test_trunc(self):
        self.ioctx.write("abc", "abc")
        self.ioctx.trunc("abc", 2)
        eq(self.ioctx.read("abc"), "ab")
        size = self.ioctx.stat("abc")[0]
        eq(size, 2)

    def test_list_objects_empty(self):
        eq(list(self.ioctx.list_objects()), [])

    def test_list_objects(self):
        self.ioctx.write("a", "")
        self.ioctx.write("b", "foo")
        self.ioctx.write_full("c", "bar")
        self.ioctx.append("d", "jazz")
        object_names = [obj.key for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), ["a", "b", "c", "d"])

    def test_list_ns_objects(self):
        self.ioctx.write("a", "")
        self.ioctx.write("b", "foo")
        self.ioctx.write_full("c", "bar")
        self.ioctx.append("d", "jazz")
        self.ioctx.set_namespace("ns1")
        self.ioctx.write("ns1-a", "")
        self.ioctx.write("ns1-b", "foo")
        self.ioctx.write_full("ns1-c", "bar")
        self.ioctx.append("ns1-d", "jazz")
        self.ioctx.append("d", "jazz")
        self.ioctx.set_namespace(LIBRADOS_ALL_NSPACES)
        object_names = [(obj.nspace, obj.key) for obj in self.ioctx.list_objects()]
        eq(
            sorted(object_names),
            [
                ("", "a"),
                ("", "b"),
                ("", "c"),
                ("", "d"),
                ("ns1", "d"),
                ("ns1", "ns1-a"),
                ("ns1", "ns1-b"),
                ("ns1", "ns1-c"),
                ("ns1", "ns1-d"),
            ],
        )

    def test_xattrs(self):
        xattrs = dict(a="1", b="2", c="3", d="a\0b", e="\0")
        self.ioctx.write("abc", "")
        for key, value in xattrs.iteritems():
            self.ioctx.set_xattr("abc", key, value)
            eq(self.ioctx.get_xattr("abc", key), value)
        stored_xattrs = {}
        for key, value in self.ioctx.get_xattrs("abc"):
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_obj_xattrs(self):
        xattrs = dict(a="1", b="2", c="3", d="a\0b", e="\0")
        self.ioctx.write("abc", "")
        obj = list(self.ioctx.list_objects())[0]
        for key, value in xattrs.iteritems():
            obj.set_xattr(key, value)
            eq(obj.get_xattr(key), value)
        stored_xattrs = {}
        for key, value in obj.get_xattrs():
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_create_snap(self):
        assert_raises(ObjectNotFound, self.ioctx.remove_snap, "foo")
        self.ioctx.create_snap("foo")
        self.ioctx.remove_snap("foo")

    def test_list_snaps_empty(self):
        eq(list(self.ioctx.list_snaps()), [])

    def test_list_snaps(self):
        snaps = ["snap1", "snap2", "snap3"]
        for snap in snaps:
            self.ioctx.create_snap(snap)
        listed_snaps = [snap.name for snap in self.ioctx.list_snaps()]
        eq(snaps, listed_snaps)

    def test_lookup_snap(self):
        self.ioctx.create_snap("foo")
        snap = self.ioctx.lookup_snap("foo")
        eq(snap.name, "foo")

    def test_snap_timestamp(self):
        self.ioctx.create_snap("foo")
        snap = self.ioctx.lookup_snap("foo")
        snap.get_timestamp()

    def test_remove_snap(self):
        self.ioctx.create_snap("foo")
        (snap,) = self.ioctx.list_snaps()
        eq(snap.name, "foo")
        self.ioctx.remove_snap("foo")
        eq(list(self.ioctx.list_snaps()), [])

    def test_set_omap(self):
        keys = ("1", "2", "3", "4")
        values = ("aaa", "bbb", "ccc", "\x04\x04\x04\x04")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 4)
            self.ioctx.operate_read_op(read_op, "hw")
            iter.next()
            eq(list(iter), [("2", "bbb"), ("3", "ccc"), ("4", "\x04\x04\x04\x04")])

    def test_get_omap_vals_by_keys(self):
        keys = ("1", "2", "3", "4")
        values = ("aaa", "bbb", "ccc", "\x04\x04\x04\x04")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals_by_keys(read_op, ("3", "4"))
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [("3", "ccc"), ("4", "\x04\x04\x04\x04")])

    def test_get_omap_keys(self):
        keys = ("1", "2", "3")
        values = ("aaa", "bbb", "ccc")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_keys(read_op, "", 2)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [("1", None), ("2", None)])

    def test_clear_omap(self):
        keys = ("1", "2", "3")
        values = ("aaa", "bbb", "ccc")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with WriteOpCtx(self.ioctx) as write_op_1:
            self.ioctx.clear_omap(write_op_1)
            self.ioctx.operate_write_op(write_op_1, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals_by_keys(read_op, ("1",))
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [])

    def test_locator(self):
        self.ioctx.set_locator_key("bar")
        self.ioctx.write("foo", "contents1")
        objects = [i for i in self.ioctx.list_objects()]
        eq(len(objects), 1)
        eq(self.ioctx.get_locator_key(), "bar")
        self.ioctx.set_locator_key("")
        objects[0].seek(0)
        objects[0].write("contents2")
        eq(self.ioctx.get_locator_key(), "")
        self.ioctx.set_locator_key("bar")
        contents = self.ioctx.read("foo")
        eq(contents, "contents2")
        eq(self.ioctx.get_locator_key(), "bar")
        objects[0].remove()
        objects = [i for i in self.ioctx.list_objects()]
        eq(objects, [])
        self.ioctx.set_locator_key("")

    def test_aio_write(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        comp = self.ioctx.aio_write("foo", "bar", 0, cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, "bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_append(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        comp = self.ioctx.aio_write("foo", "bar", 0, cb, cb)
        comp2 = self.ioctx.aio_append("foo", "baz", cb, cb)
        comp.wait_for_complete()
        contents = self.ioctx.read("foo")
        eq(contents, "barbaz")
        with lock:
            while count[0] < 4:
                lock.wait()
        eq(comp.get_return_value(), 0)
        eq(comp2.get_return_value(), 0)
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_write_full(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        self.ioctx.aio_write("foo", "barbaz", 0, cb, cb)
        comp = self.ioctx.aio_write_full("foo", "bar", cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, "bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def _take_down_acting_set(self, pool, objectname):
        # find acting_set for pool:objectname and take it down; used to
        # verify that async reads don't complete while acting set is missing
        cmd = {"prefix": "osd map", "pool": pool, "object": objectname, "format": "json"}
        r, jsonout, _ = self.rados.mon_command(json.dumps(cmd), "")
        objmap = json.loads(jsonout)
        acting_set = objmap["acting"]
        cmd = {"prefix": "osd set", "key": "noup"}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), "")
        eq(r, 0)
        cmd = {"prefix": "osd down", "ids": [str(i) for i in acting_set]}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), "")
        eq(r, 0)

        # wait for OSDs to acknowledge the down
        eq(self.rados.wait_for_latest_osdmap(), 0)

    def _let_osds_back_up(self):
        cmd = {"prefix": "osd unset", "key": "noup"}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), "")
        eq(r, 0)

    def test_aio_read(self):
        # this is a list so that the local cb() can modify it
        retval = [None]
        lock = threading.Condition()

        def cb(_, buf):
            with lock:
                retval[0] = buf
                lock.notify()

        payload = "bar\000frob"
        self.ioctx.write("foo", payload)

        # test1: use wait_for_complete() and wait for cb by
        # watching retval[0]
        self._take_down_acting_set("test_pool", "foo")
        comp = self.ioctx.aio_read("foo", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(None, retval[0])
        self._let_osds_back_up()
        comp.wait_for_complete()
        loops = 0
        with lock:
            while retval[0] is None and loops <= 10:
                lock.wait(timeout=5)
                loops += 1
        assert loops <= 10

        eq(retval[0], payload)

        # test2: use wait_for_complete_and_cb(), verify retval[0] is
        # set by the time we regain control

        retval[0] = None
        self._take_down_acting_set("test_pool", "foo")
        comp = self.ioctx.aio_read("foo", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(None, retval[0])
        self._let_osds_back_up()

        comp.wait_for_complete_and_cb()
        assert retval[0] is not None
        eq(retval[0], payload)

        [i.remove() for i in self.ioctx.list_objects()]

    def test_lock(self):
        self.ioctx.lock_exclusive("foo", "lock", "locker", "desc_lock", 10000, 0)
        assert_raises(ObjectExists, self.ioctx.lock_exclusive, "foo", "lock", "locker", "desc_lock", 10000, 0)
        self.ioctx.unlock("foo", "lock", "locker")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock", "locker")

        self.ioctx.lock_shared("foo", "lock", "locker1", "tag", "desc_lock", 10000, 0)
        self.ioctx.lock_shared("foo", "lock", "locker2", "tag", "desc_lock", 10000, 0)
        assert_raises(ObjectBusy, self.ioctx.lock_exclusive, "foo", "lock", "locker3", "desc_lock", 10000, 0)
        self.ioctx.unlock("foo", "lock", "locker1")
        self.ioctx.unlock("foo", "lock", "locker2")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock", "locker1")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock", "locker2")
Beispiel #24
0
class TestIoctx(object):

    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')

    def tearDown(self):
        cmd = {"prefix":"osd unset", "key":"noup"}
        self.rados.mon_command(json.dumps(cmd), b'')
        self.ioctx.close()
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()

    def test_get_last_version(self):
        version = self.ioctx.get_last_version()
        assert version >= 0

    def test_get_stats(self):
        stats = self.ioctx.get_stats()
        eq(stats, {'num_objects_unfound': 0,
                   'num_objects_missing_on_primary': 0,
                   'num_object_clones': 0,
                   'num_objects': 0,
                   'num_object_copies': 0,
                   'num_bytes': 0,
                   'num_rd_kb': 0,
                   'num_wr_kb': 0,
                   'num_kb': 0,
                   'num_wr': 0,
                   'num_objects_degraded': 0,
                   'num_rd': 0})

    def test_change_auid(self):
        self.ioctx.change_auid(ANONYMOUS_AUID)
        self.ioctx.change_auid(ADMIN_AUID)

    def test_write(self):
        self.ioctx.write('abc', b'abc')
        eq(self.ioctx.read('abc'), b'abc')

    def test_write_full(self):
        self.ioctx.write('abc', b'abc')
        eq(self.ioctx.read('abc'), b'abc')
        self.ioctx.write_full('abc', b'd')
        eq(self.ioctx.read('abc'), b'd')

    def test_append(self):
        self.ioctx.write('abc', b'a')
        self.ioctx.append('abc', b'b')
        self.ioctx.append('abc', b'c')
        eq(self.ioctx.read('abc'), b'abc')

    def test_write_zeros(self):
        self.ioctx.write('abc', b'a\0b\0c')
        eq(self.ioctx.read('abc'), b'a\0b\0c')

    def test_trunc(self):
        self.ioctx.write('abc', b'abc')
        self.ioctx.trunc('abc', 2)
        eq(self.ioctx.read('abc'), b'ab')
        size = self.ioctx.stat('abc')[0]
        eq(size, 2)

    def test_list_objects_empty(self):
        eq(list(self.ioctx.list_objects()), [])

    def test_list_objects(self):
        self.ioctx.write('a', b'')
        self.ioctx.write('b', b'foo')
        self.ioctx.write_full('c', b'bar')
        self.ioctx.append('d', b'jazz')
        object_names = [obj.key for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), ['a', 'b', 'c', 'd'])

    def test_list_ns_objects(self):
        self.ioctx.write('a', b'')
        self.ioctx.write('b', b'foo')
        self.ioctx.write_full('c', b'bar')
        self.ioctx.append('d', b'jazz')
        self.ioctx.set_namespace("ns1")
        self.ioctx.write('ns1-a', b'')
        self.ioctx.write('ns1-b', b'foo')
        self.ioctx.write_full('ns1-c', b'bar')
        self.ioctx.append('ns1-d', b'jazz')
        self.ioctx.append('d', b'jazz')
        self.ioctx.set_namespace(LIBRADOS_ALL_NSPACES)
        object_names = [(obj.nspace, obj.key) for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), [('', 'a'), ('','b'), ('','c'), ('','d'),\
                ('ns1', 'd'), ('ns1', 'ns1-a'), ('ns1', 'ns1-b'),\
                ('ns1', 'ns1-c'), ('ns1', 'ns1-d')])

    def test_xattrs(self):
        xattrs = dict(a=b'1', b=b'2', c=b'3', d=b'a\0b', e=b'\0')
        self.ioctx.write('abc', b'')
        for key, value in xattrs.items():
            self.ioctx.set_xattr('abc', key, value)
            eq(self.ioctx.get_xattr('abc', key), value)
        stored_xattrs = {}
        for key, value in self.ioctx.get_xattrs('abc'):
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_obj_xattrs(self):
        xattrs = dict(a=b'1', b=b'2', c=b'3', d=b'a\0b', e=b'\0')
        self.ioctx.write('abc', b'')
        obj = list(self.ioctx.list_objects())[0]
        for key, value in xattrs.items():
            obj.set_xattr(key, value)
            eq(obj.get_xattr(key), value)
        stored_xattrs = {}
        for key, value in obj.get_xattrs():
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_create_snap(self):
        assert_raises(ObjectNotFound, self.ioctx.remove_snap, 'foo')
        self.ioctx.create_snap('foo')
        self.ioctx.remove_snap('foo')

    def test_list_snaps_empty(self):
        eq(list(self.ioctx.list_snaps()), [])

    def test_list_snaps(self):
        snaps = ['snap1', 'snap2', 'snap3']
        for snap in snaps:
            self.ioctx.create_snap(snap)
        listed_snaps = [snap.name for snap in self.ioctx.list_snaps()]
        eq(snaps, listed_snaps)

    def test_lookup_snap(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        eq(snap.name, 'foo')

    def test_snap_timestamp(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        snap.get_timestamp()

    def test_remove_snap(self):
        self.ioctx.create_snap('foo')
        (snap,) = self.ioctx.list_snaps()
        eq(snap.name, 'foo')
        self.ioctx.remove_snap('foo')
        eq(list(self.ioctx.list_snaps()), [])

    def test_snap_rollback(self):
        self.ioctx.write("insnap", b"contents1")
        self.ioctx.create_snap("snap1")
        self.ioctx.remove_object("insnap")
        self.ioctx.snap_rollback("insnap", "snap1")
        eq(self.ioctx.read("insnap"), b"contents1")
        self.ioctx.remove_snap("snap1")
        self.ioctx.remove_object("insnap")

    def test_snap_read(self):
        self.ioctx.write("insnap", b"contents1")
        self.ioctx.create_snap("snap1")
        self.ioctx.remove_object("insnap")
        snap = self.ioctx.lookup_snap("snap1")
        self.ioctx.set_read(snap.snap_id)
        eq(self.ioctx.read("insnap"), b"contents1")
        self.ioctx.set_read(LIBRADOS_SNAP_HEAD)
        self.ioctx.write("inhead", b"contents2")
        eq(self.ioctx.read("inhead"), b"contents2")
        self.ioctx.remove_snap("snap1")
        self.ioctx.remove_object("inhead")

    def test_set_omap(self):
        keys = ("1", "2", "3", "4")
        values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            write_op.set_flags(LIBRADOS_OPERATION_SKIPRWLOCKS)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 4)
            eq(ret, 0)
            self.ioctx.operate_read_op(read_op, "hw")
            next(iter)
            eq(list(iter), [("2", b"bbb"), ("3", b"ccc"), ("4", b"\x04\x04\x04\x04")])
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals(read_op, "2", "", 4)
            eq(ret, 0)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(("3", b"ccc"), next(iter))
            eq(list(iter), [("4", b"\x04\x04\x04\x04")])
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals(read_op, "", "2", 4)
            eq(ret, 0)
            read_op.set_flags(LIBRADOS_OPERATION_BALANCE_READS)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [("2", b"bbb")])

    def test_set_omap_aio(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        keys = ("1", "2", "3", "4")
        values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            comp = self.ioctx.operate_aio_write_op(write_op, "hw", cb, cb)
            comp.wait_for_complete()
            comp.wait_for_safe()
            with lock:
                while count[0] < 2:
                    lock.wait()
            eq(comp.get_return_value(), 0)

        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 4)
            eq(ret, 0)
            comp = self.ioctx.operate_aio_read_op(read_op, "hw", cb, cb)
            comp.wait_for_complete()
            comp.wait_for_safe()
            with lock:
                while count[0] < 4:
                    lock.wait()
            eq(comp.get_return_value(), 0)
            next(iter)
            eq(list(iter), [("2", b"bbb"), ("3", b"ccc"), ("4", b"\x04\x04\x04\x04")])

    def test_write_ops(self):
        with WriteOpCtx(self.ioctx) as write_op:
            write_op.new(0)
            self.ioctx.operate_write_op(write_op, "write_ops")
            eq(self.ioctx.read('write_ops'), b'')
            write_op.write_full(b'1')
            write_op.append(b'2')
            self.ioctx.operate_write_op(write_op, "write_ops")
            eq(self.ioctx.read('write_ops'), b'12')
            write_op.write_full(b'12345')
            write_op.write(b'x', 2)
            self.ioctx.operate_write_op(write_op, "write_ops")
            eq(self.ioctx.read('write_ops'), b'12x45')
            write_op.write_full(b'12345')
            write_op.zero(2, 2)
            self.ioctx.operate_write_op(write_op, "write_ops")
            eq(self.ioctx.read('write_ops'), b'12\x00\x005')
            write_op.write_full(b'12345')
            write_op.truncate(2)
            self.ioctx.operate_write_op(write_op, "write_ops")
            eq(self.ioctx.read('write_ops'), b'12')
            write_op.remove()
            self.ioctx.operate_write_op(write_op, "write_ops")
            with assert_raises(ObjectNotFound):
                self.ioctx.read('write_ops')

    def test_get_omap_vals_by_keys(self):
        keys = ("1", "2", "3", "4")
        values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3","4",))
            eq(ret, 0)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [("3", b"ccc"), ("4", b"\x04\x04\x04\x04")])
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3","4",))
            eq(ret, 0)
            with assert_raises(ObjectNotFound):
                self.ioctx.operate_read_op(read_op, "no_such")

    def test_get_omap_keys(self):
        keys = ("1", "2", "3")
        values = (b"aaa", b"bbb", b"ccc")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_keys(read_op,"",2)
            eq(ret, 0)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [("1", None), ("2", None)])
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_keys(read_op,"",2)
            eq(ret, 0)
            with assert_raises(ObjectNotFound):
                self.ioctx.operate_read_op(read_op, "no_such")

    def test_clear_omap(self):
        keys = ("1", "2", "3")
        values = (b"aaa", b"bbb", b"ccc")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with WriteOpCtx(self.ioctx) as write_op_1:
            self.ioctx.clear_omap(write_op_1)
            self.ioctx.operate_write_op(write_op_1, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("1",))
            eq(ret, 0)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [])

    def test_locator(self):
        self.ioctx.set_locator_key("bar")
        self.ioctx.write('foo', b'contents1')
        objects = [i for i in self.ioctx.list_objects()]
        eq(len(objects), 1)
        eq(self.ioctx.get_locator_key(), "bar")
        self.ioctx.set_locator_key("")
        objects[0].seek(0)
        objects[0].write(b"contents2")
        eq(self.ioctx.get_locator_key(), "")
        self.ioctx.set_locator_key("bar")
        contents = self.ioctx.read("foo")
        eq(contents, b"contents2")
        eq(self.ioctx.get_locator_key(), "bar")
        objects[0].remove()
        objects = [i for i in self.ioctx.list_objects()]
        eq(objects, [])
        self.ioctx.set_locator_key("")

    def test_aio_write(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0
        comp = self.ioctx.aio_write("foo", b"bar", 0, cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, b"bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_write_no_comp_ref(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0
        # NOTE(sileht): force don't save the comp into local var
        # to ensure all references are correctly tracked into the lib
        self.ioctx.aio_write("foo", b"bar", 0, cb, cb)
        with lock:
            while count[0] < 2:
                lock.wait()
        contents = self.ioctx.read("foo")
        eq(contents, b"bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_append(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0
        comp = self.ioctx.aio_write("foo", b"bar", 0, cb, cb)
        comp2 = self.ioctx.aio_append("foo", b"baz", cb, cb)
        comp.wait_for_complete()
        contents = self.ioctx.read("foo")
        eq(contents, b"barbaz")
        with lock:
            while count[0] < 4:
                lock.wait()
        eq(comp.get_return_value(), 0)
        eq(comp2.get_return_value(), 0)
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_write_full(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0
        self.ioctx.aio_write("foo", b"barbaz", 0, cb, cb)
        comp = self.ioctx.aio_write_full("foo", b"bar", cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, b"bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def _take_down_acting_set(self, pool, objectname):
        # find acting_set for pool:objectname and take it down; used to
        # verify that async reads don't complete while acting set is missing
        cmd = {
            "prefix":"osd map",
            "pool":pool,
            "object":objectname,
            "format":"json",
        }
        r, jsonout, _ = self.rados.mon_command(json.dumps(cmd), b'')
        objmap = json.loads(jsonout.decode("utf-8"))
        acting_set = objmap['acting']
        cmd = {"prefix":"osd set", "key":"noup"}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), b'')
        eq(r, 0)
        cmd = {"prefix":"osd down", "ids":[str(i) for i in acting_set]}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), b'')
        eq(r, 0)

        # wait for OSDs to acknowledge the down
        eq(self.rados.wait_for_latest_osdmap(), 0)

    def _let_osds_back_up(self):
        cmd = {"prefix":"osd unset", "key":"noup"}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), b'')
        eq(r, 0)

    def test_aio_read(self):
        # this is a list so that the local cb() can modify it
        retval = [None]
        lock = threading.Condition()
        def cb(_, buf):
            with lock:
                retval[0] = buf
                lock.notify()
        payload = b"bar\000frob"
        self.ioctx.write("foo", payload)

        # test1: use wait_for_complete() and wait for cb by
        # watching retval[0]
        self._take_down_acting_set('test_pool', 'foo')
        comp = self.ioctx.aio_read("foo", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(None, retval[0])
        self._let_osds_back_up()
        comp.wait_for_complete()
        loops = 0
        with lock:
            while retval[0] is None and loops <= 10:
                lock.wait(timeout=5)
                loops += 1
        assert(loops <= 10)

        eq(retval[0], payload)
        eq(sys.getrefcount(comp), 2)

        # test2: use wait_for_complete_and_cb(), verify retval[0] is
        # set by the time we regain control

        retval[0] = None
        self._take_down_acting_set('test_pool', 'foo')
        comp = self.ioctx.aio_read("foo", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(None, retval[0])
        self._let_osds_back_up()

        comp.wait_for_complete_and_cb()
        assert(retval[0] is not None)
        eq(retval[0], payload)
        eq(sys.getrefcount(comp), 2)

        # test3: error case, use wait_for_complete_and_cb(), verify retval[0] is
        # set by the time we regain control

        retval[0] = 1
        self._take_down_acting_set('test_pool', 'bar')
        comp = self.ioctx.aio_read("bar", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(1, retval[0])
        self._let_osds_back_up()

        comp.wait_for_complete_and_cb()
        eq(None, retval[0])
        assert(comp.get_return_value() < 0)
        eq(sys.getrefcount(comp), 2)

        [i.remove() for i in self.ioctx.list_objects()]

    def test_lock(self):
        self.ioctx.lock_exclusive("foo", "lock", "locker", "desc_lock",
                                  10000, 0)
        assert_raises(ObjectExists,
                      self.ioctx.lock_exclusive,
                      "foo", "lock", "locker", "desc_lock", 10000, 0)
        self.ioctx.unlock("foo", "lock", "locker")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock", "locker")

        self.ioctx.lock_shared("foo", "lock", "locker1", "tag", "desc_lock",
                               10000, 0)
        self.ioctx.lock_shared("foo", "lock", "locker2", "tag", "desc_lock",
                               10000, 0)
        assert_raises(ObjectBusy,
                      self.ioctx.lock_exclusive,
                      "foo", "lock", "locker3", "desc_lock", 10000, 0)
        self.ioctx.unlock("foo", "lock", "locker1")
        self.ioctx.unlock("foo", "lock", "locker2")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock", "locker1")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock", "locker2")

    def test_execute(self):
        self.ioctx.write("foo", b"") # ensure object exists

        ret, buf = self.ioctx.execute("foo", "hello", "say_hello", b"")
        eq(buf, b"Hello, world!")

        ret, buf = self.ioctx.execute("foo", "hello", "say_hello", b"nose")
        eq(buf, b"Hello, nose!")
Beispiel #25
0
class TestIoctx(object):

    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')

    def tearDown(self):
        cmd = {"prefix":"osd unset", "key":"noup"}
        self.rados.mon_command(json.dumps(cmd), '')
        self.ioctx.close()
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()

    def test_change_auid(self):
        self.ioctx.change_auid(ANONYMOUS_AUID)
        self.ioctx.change_auid(ADMIN_AUID)

    def test_write(self):
        self.ioctx.write('abc', 'abc')
        eq(self.ioctx.read('abc'), 'abc')

    def test_write_full(self):
        self.ioctx.write('abc', 'abc')
        eq(self.ioctx.read('abc'), 'abc')
        self.ioctx.write_full('abc', 'd')
        eq(self.ioctx.read('abc'), 'd')

    def test_append(self):
        self.ioctx.write('abc', 'a')
        self.ioctx.append('abc', 'b')
        self.ioctx.append('abc', 'c')
        eq(self.ioctx.read('abc'), 'abc')

    def test_write_zeros(self):
        self.ioctx.write('abc', 'a\0b\0c')
        eq(self.ioctx.read('abc'), 'a\0b\0c')

    def test_trunc(self):
        self.ioctx.write('abc', 'abc')
        self.ioctx.trunc('abc', 2)
        eq(self.ioctx.read('abc'), 'ab')
        size = self.ioctx.stat('abc')[0]
        eq(size, 2)

    def test_list_objects_empty(self):
        eq(list(self.ioctx.list_objects()), [])

    def test_list_objects(self):
        self.ioctx.write('a', '')
        self.ioctx.write('b', 'foo')
        self.ioctx.write_full('c', 'bar')
        self.ioctx.append('d', 'jazz')
        object_names = [obj.key for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), ['a', 'b', 'c', 'd'])

    def test_list_ns_objects(self):
        self.ioctx.write('a', '')
        self.ioctx.write('b', 'foo')
        self.ioctx.write_full('c', 'bar')
        self.ioctx.append('d', 'jazz')
        self.ioctx.set_namespace("ns1")
        self.ioctx.write('ns1-a', '')
        self.ioctx.write('ns1-b', 'foo')
        self.ioctx.write_full('ns1-c', 'bar')
        self.ioctx.append('ns1-d', 'jazz')
        self.ioctx.append('d', 'jazz')
        self.ioctx.set_namespace(LIBRADOS_ALL_NSPACES)
        object_names = [(obj.nspace, obj.key) for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), [('', 'a'), ('','b'), ('','c'), ('','d'),\
                ('ns1', 'd'), ('ns1', 'ns1-a'), ('ns1', 'ns1-b'),\
                ('ns1', 'ns1-c'), ('ns1', 'ns1-d')])

    def test_xattrs(self):
        xattrs = dict(a='1', b='2', c='3', d='a\0b', e='\0')
        self.ioctx.write('abc', '')
        for key, value in xattrs.iteritems():
            self.ioctx.set_xattr('abc', key, value)
            eq(self.ioctx.get_xattr('abc', key), value)
        stored_xattrs = {}
        for key, value in self.ioctx.get_xattrs('abc'):
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_obj_xattrs(self):
        xattrs = dict(a='1', b='2', c='3', d='a\0b', e='\0')
        self.ioctx.write('abc', '')
        obj = list(self.ioctx.list_objects())[0]
        for key, value in xattrs.iteritems():
            obj.set_xattr(key, value)
            eq(obj.get_xattr(key), value)
        stored_xattrs = {}
        for key, value in obj.get_xattrs():
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_create_snap(self):
        assert_raises(ObjectNotFound, self.ioctx.remove_snap, 'foo')
        self.ioctx.create_snap('foo')
        self.ioctx.remove_snap('foo')

    def test_list_snaps_empty(self):
        eq(list(self.ioctx.list_snaps()), [])

    def test_list_snaps(self):
        snaps = ['snap1', 'snap2', 'snap3']
        for snap in snaps:
            self.ioctx.create_snap(snap)
        listed_snaps = [snap.name for snap in self.ioctx.list_snaps()]
        eq(snaps, listed_snaps)

    def test_lookup_snap(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        eq(snap.name, 'foo')

    def test_snap_timestamp(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        snap.get_timestamp()

    def test_remove_snap(self):
        self.ioctx.create_snap('foo')
        (snap,) = self.ioctx.list_snaps()
        eq(snap.name, 'foo')
        self.ioctx.remove_snap('foo')
        eq(list(self.ioctx.list_snaps()), [])

    def test_locator(self):
        self.ioctx.set_locator_key("bar")
        self.ioctx.write('foo', 'contents1')
        objects = [i for i in self.ioctx.list_objects()]
        eq(len(objects), 1)
        eq(self.ioctx.get_locator_key(), "bar")
        self.ioctx.set_locator_key("")
        objects[0].seek(0)
        objects[0].write("contents2")
        eq(self.ioctx.get_locator_key(), "")
        self.ioctx.set_locator_key("bar")
        contents = self.ioctx.read("foo")
        eq(contents, "contents2")
        eq(self.ioctx.get_locator_key(), "bar")
        objects[0].remove()
        objects = [i for i in self.ioctx.list_objects()]
        eq(objects, [])
        self.ioctx.set_locator_key("")

    def test_aio_write(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0
        comp = self.ioctx.aio_write("foo", "bar", 0, cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, "bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_append(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0
        comp = self.ioctx.aio_write("foo", "bar", 0, cb, cb)
        comp2 = self.ioctx.aio_append("foo", "baz", cb, cb)
        comp.wait_for_complete()
        contents = self.ioctx.read("foo")
        eq(contents, "barbaz")
        with lock:
            while count[0] < 4:
                lock.wait()
        eq(comp.get_return_value(), 0)
        eq(comp2.get_return_value(), 0)
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_write_full(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0
        self.ioctx.aio_write("foo", "barbaz", 0, cb, cb)
        comp = self.ioctx.aio_write_full("foo", "bar", cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, "bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def _take_down_acting_set(self, pool, objectname):
        # find acting_set for pool:objectname and take it down; used to
        # verify that async reads don't complete while acting set is missing
        cmd = {
            "prefix":"osd map",
            "pool":pool,
            "object":objectname,
            "format":"json",
        }
        r, jsonout, _ = self.rados.mon_command(json.dumps(cmd), '')
        objmap = json.loads(jsonout)
        acting_set = objmap['acting']
        cmd = {"prefix":"osd set", "key":"noup"}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), '')
        eq(r, 0)
        cmd = {"prefix":"osd down", "ids":[str(i) for i in acting_set]}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), '')
        eq(r, 0)

        # wait for OSDs to acknowledge the down
        eq(self.rados.wait_for_latest_osdmap(), 0)

    def _let_osds_back_up(self):
        cmd = {"prefix":"osd unset", "key":"noup"}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), '')
        eq(r, 0)

    def test_aio_read(self):
        # this is a list so that the local cb() can modify it
        retval = [None]
        lock = threading.Condition()
        def cb(_, buf):
            with lock:
                retval[0] = buf
                lock.notify()
        payload = "bar\000frob"
        self.ioctx.write("foo", payload)

        # test1: use wait_for_complete() and wait for cb by
        # watching retval[0]
        self._take_down_acting_set('test_pool', 'foo')
        comp = self.ioctx.aio_read("foo", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(None, retval[0])
        self._let_osds_back_up()
        comp.wait_for_complete()
        loops = 0
        with lock:
            while retval[0] is None and loops <= 10:
                lock.wait(timeout=5)
                loops += 1
        assert(loops <= 10)

        eq(retval[0], payload)

        # test2: use wait_for_complete_and_cb(), verify retval[0] is
        # set by the time we regain control

        retval[0] = None
        self._take_down_acting_set('test_pool', 'foo')
        comp = self.ioctx.aio_read("foo", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(None, retval[0])
        self._let_osds_back_up()

        comp.wait_for_complete_and_cb()
        assert(retval[0] is not None)
        eq(retval[0], payload)

        [i.remove() for i in self.ioctx.list_objects()]
Beispiel #26
0
class TestIoctx(object):
    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')

    def tearDown(self):
        self.ioctx.close()
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()

    def test_change_auid(self):
        self.ioctx.change_auid(ANONYMOUS_AUID)
        self.ioctx.change_auid(ADMIN_AUID)

    def test_write(self):
        self.ioctx.write('abc', 'abc')
        eq(self.ioctx.read('abc'), 'abc')

    def test_write_full(self):
        self.ioctx.write('abc', 'abc')
        eq(self.ioctx.read('abc'), 'abc')
        self.ioctx.write_full('abc', 'd')
        eq(self.ioctx.read('abc'), 'd')

    def test_write_zeros(self):
        self.ioctx.write('abc', 'a\0b\0c')
        eq(self.ioctx.read('abc'), 'a\0b\0c')

    def test_list_objects_empty(self):
        eq(list(self.ioctx.list_objects()), [])

    def test_list_objects(self):
        self.ioctx.write('a', '')
        self.ioctx.write('b', 'foo')
        self.ioctx.write_full('c', 'bar')
        object_names = [obj.key for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), ['a', 'b', 'c'])

    def test_xattrs(self):
        xattrs = dict(a='1', b='2', c='3', d='a\0b', e='\0')
        self.ioctx.write('abc', '')
        for key, value in xattrs.iteritems():
            self.ioctx.set_xattr('abc', key, value)
            eq(self.ioctx.get_xattr('abc', key), value)
        stored_xattrs = {}
        for key, value in self.ioctx.get_xattrs('abc'):
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_create_snap(self):
        assert_raises(ObjectNotFound, self.ioctx.remove_snap, 'foo')
        self.ioctx.create_snap('foo')
        self.ioctx.remove_snap('foo')

    def test_list_snaps_empty(self):
        eq(list(self.ioctx.list_snaps()), [])

    def test_list_snaps(self):
        snaps = ['snap1', 'snap2', 'snap3']
        for snap in snaps:
            self.ioctx.create_snap(snap)
        listed_snaps = [snap.name for snap in self.ioctx.list_snaps()]
        eq(snaps, listed_snaps)

    def test_lookup_snap(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        eq(snap.name, 'foo')

    def test_snap_timestamp(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        snap.get_timestamp()

    def test_remove_snap(self):
        self.ioctx.create_snap('foo')
        (snap, ) = self.ioctx.list_snaps()
        eq(snap.name, 'foo')
        self.ioctx.remove_snap('foo')
        eq(list(self.ioctx.list_snaps()), [])

    def test_locator(self):
        self.ioctx.set_locator_key("bar")
        self.ioctx.write('foo', 'contents1')
        objects = [i for i in self.ioctx.list_objects()]
        eq(len(objects), 1)
        eq(self.ioctx.get_locator_key(), "bar")
        self.ioctx.set_locator_key("")
        objects[0].seek(0)
        objects[0].write("contents2")
        eq(self.ioctx.get_locator_key(), "")
        self.ioctx.set_locator_key("bar")
        contents = self.ioctx.read("foo")
        eq(contents, "contents2")
        eq(self.ioctx.get_locator_key(), "bar")
        objects[0].remove()
        objects = [i for i in self.ioctx.list_objects()]
        eq(objects, [])
        self.ioctx.set_locator_key("")

    def test_aio_write(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        comp = self.ioctx.aio_write("foo", "bar", 0, cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, "bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_append(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        comp = self.ioctx.aio_write("foo", "bar", 0, cb, cb)
        comp2 = self.ioctx.aio_append("foo", "baz", cb, cb)
        comp.wait_for_complete()
        contents = self.ioctx.read("foo")
        eq(contents, "barbaz")
        with lock:
            while count[0] < 4:
                lock.wait()
        eq(comp.get_return_value(), 0)
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_write_full(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        self.ioctx.aio_write("foo", "barbaz", 0, cb, cb)
        comp = self.ioctx.aio_write_full("foo", "bar", cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, "bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_read(self):
        retval = [None]
        lock = threading.Condition()

        def cb(_, buf):
            with lock:
                retval[0] = buf
                lock.notify()

        self.ioctx.write("foo", "bar")
        self.ioctx.aio_read("foo", 3, 0, cb)
        with lock:
            while retval[0] is None:
                lock.wait()
        eq(retval[0], "bar")
        [i.remove() for i in self.ioctx.list_objects()]
Beispiel #27
0
class TestWatchNotify(object):
    OID = "test_watch_notify"

    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')
        self.ioctx.write(self.OID, b'test watch notify')
        self.lock = threading.Condition()
        self.notify_cnt = {}
        self.notify_data = {}
        self.notify_error = {}

    def tearDown(self):
        self.ioctx.close()
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()

    def make_callback(self):
        def callback(notify_id, notifier_id, watch_id, data):
            with self.lock:
                if watch_id not in self.notify_cnt:
                    self.notify_cnt[watch_id] = 0
                self.notify_cnt[watch_id] += 1
                self.notify_data[watch_id] = data
        return callback

    def make_error_callback(self):
        def callback(watch_id, error):
            with self.lock:
                self.notify_error[watch_id] = error
        return callback


    def test(self):
        with self.ioctx.watch(self.OID, self.make_callback(),
                              self.make_error_callback()) as watch1:
            watch_id1 = watch1.get_id()
            assert(watch_id1 > 0)

            with self.rados.open_ioctx('test_pool') as ioctx:
                watch2 = ioctx.watch(self.OID, self.make_callback(),
                                     self.make_error_callback())
            watch_id2 = watch2.get_id()
            assert(watch_id2 > 0)

            assert(self.ioctx.notify(self.OID, 'test'))
            with self.lock:
                assert(watch_id1 in self.notify_cnt)
                assert(watch_id2 in self.notify_cnt)
                eq(self.notify_cnt[watch_id1], 1)
                eq(self.notify_cnt[watch_id2], 1)
                eq(self.notify_data[watch_id1], b'test')
                eq(self.notify_data[watch_id2], b'test')

            assert(watch1.check() >= timedelta())
            assert(watch2.check() >= timedelta())

            assert(self.ioctx.notify(self.OID, 'best'))
            with self.lock:
                eq(self.notify_cnt[watch_id1], 2)
                eq(self.notify_cnt[watch_id2], 2)
                eq(self.notify_data[watch_id1], b'best')
                eq(self.notify_data[watch_id2], b'best')

            watch2.close()

            assert(self.ioctx.notify(self.OID, 'rest'))
            with self.lock:
                eq(self.notify_cnt[watch_id1], 3)
                eq(self.notify_cnt[watch_id2], 2)
                eq(self.notify_data[watch_id1], b'rest')
                eq(self.notify_data[watch_id2], b'best')

            assert(watch1.check() >= timedelta())

            self.ioctx.remove_object(self.OID)

            for i in range(10):
                with self.lock:
                    if watch_id1 in self.notify_error:
                        break
                time.sleep(1)
            eq(self.notify_error[watch_id1], -errno.ENOTCONN)
            assert_raises(NotConnected, watch1.check)

            assert_raises(ObjectNotFound, self.ioctx.notify, self.OID, 'test')
Beispiel #28
0
class TestWatchNotify(object):
    OID = "test_watch_notify"

    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')
        self.ioctx.write(self.OID, b'test watch notify')
        self.lock = threading.Condition()
        self.notify_cnt = {}
        self.notify_data = {}
        self.notify_error = {}
        # aio related
        self.ack_cnt = {}
        self.ack_data = {}
        self.instance_id = self.rados.get_instance_id()

    def tearDown(self):
        self.ioctx.close()
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()

    def make_callback(self):
        def callback(notify_id, notifier_id, watch_id, data):
            with self.lock:
                if watch_id not in self.notify_cnt:
                    self.notify_cnt[watch_id] = 0
                self.notify_cnt[watch_id] += 1
                self.notify_data[watch_id] = data
        return callback

    def make_error_callback(self):
        def callback(watch_id, error):
            with self.lock:
                self.notify_error[watch_id] = error
        return callback


    def test(self):
        with self.ioctx.watch(self.OID, self.make_callback(),
                              self.make_error_callback()) as watch1:
            watch_id1 = watch1.get_id()
            assert(watch_id1 > 0)

            with self.rados.open_ioctx('test_pool') as ioctx:
                watch2 = ioctx.watch(self.OID, self.make_callback(),
                                     self.make_error_callback())
            watch_id2 = watch2.get_id()
            assert(watch_id2 > 0)

            assert(self.ioctx.notify(self.OID, 'test'))
            with self.lock:
                assert(watch_id1 in self.notify_cnt)
                assert(watch_id2 in self.notify_cnt)
                eq(self.notify_cnt[watch_id1], 1)
                eq(self.notify_cnt[watch_id2], 1)
                eq(self.notify_data[watch_id1], b'test')
                eq(self.notify_data[watch_id2], b'test')

            assert(watch1.check() >= timedelta())
            assert(watch2.check() >= timedelta())

            assert(self.ioctx.notify(self.OID, 'best'))
            with self.lock:
                eq(self.notify_cnt[watch_id1], 2)
                eq(self.notify_cnt[watch_id2], 2)
                eq(self.notify_data[watch_id1], b'best')
                eq(self.notify_data[watch_id2], b'best')

            watch2.close()

            assert(self.ioctx.notify(self.OID, 'rest'))
            with self.lock:
                eq(self.notify_cnt[watch_id1], 3)
                eq(self.notify_cnt[watch_id2], 2)
                eq(self.notify_data[watch_id1], b'rest')
                eq(self.notify_data[watch_id2], b'best')

            assert(watch1.check() >= timedelta())

            self.ioctx.remove_object(self.OID)

            for i in range(10):
                with self.lock:
                    if watch_id1 in self.notify_error:
                        break
                time.sleep(1)
            eq(self.notify_error[watch_id1], -errno.ENOTCONN)
            assert_raises(NotConnected, watch1.check)

            assert_raises(ObjectNotFound, self.ioctx.notify, self.OID, 'test')

    def make_callback_reply(self):
        def callback(notify_id, notifier_id, watch_id, data):
            with self.lock:
                return data
        return callback

    def notify_callback(self, _, r, ack_list, timeout_list):
        eq(r, 0)
        with self.lock:
            for notifier_id, _, notifier_data in ack_list:
                if notifier_id not in self.ack_cnt:
                    self.ack_cnt[notifier_id] = 0
                self.ack_cnt[notifier_id] += 1
                self.ack_data[notifier_id] = notifier_data

    def notify_callback_err(self, _, r, ack_list, timeout_list):
        eq(r, -errno.ENOENT)

    def test_aio_notify(self):
        with self.ioctx.watch(self.OID, self.make_callback_reply(),
                              self.make_error_callback()) as watch1:
            watch_id1 = watch1.get_id()
            ok(watch_id1 > 0)

            with self.rados.open_ioctx('test_pool') as ioctx:
                watch2 = ioctx.watch(self.OID, self.make_callback_reply(),
                                     self.make_error_callback())
            watch_id2 = watch2.get_id()
            ok(watch_id2 > 0)

            comp = self.ioctx.aio_notify(self.OID, self.notify_callback, msg='test')
            comp.wait_for_complete_and_cb()
            with self.lock:
                ok(self.instance_id in self.ack_cnt)
                eq(self.ack_cnt[self.instance_id], 2)
                eq(self.ack_data[self.instance_id], b'test')

            ok(watch1.check() >= timedelta())
            ok(watch2.check() >= timedelta())

            comp = self.ioctx.aio_notify(self.OID, self.notify_callback, msg='best')
            comp.wait_for_complete_and_cb()
            with self.lock:
                eq(self.ack_cnt[self.instance_id], 4)
                eq(self.ack_data[self.instance_id], b'best')

            watch2.close()

            comp = self.ioctx.aio_notify(self.OID, self.notify_callback, msg='rest')
            comp.wait_for_complete_and_cb()
            with self.lock:
                eq(self.ack_cnt[self.instance_id], 5)
                eq(self.ack_data[self.instance_id], b'rest')

            assert(watch1.check() >= timedelta())
            self.ioctx.remove_object(self.OID)

            for i in range(10):
                with self.lock:
                    if watch_id1 in self.notify_error:
                        break
                time.sleep(1)
            eq(self.notify_error[watch_id1], -errno.ENOTCONN)
            assert_raises(NotConnected, watch1.check)

            comp = self.ioctx.aio_notify(self.OID, self.notify_callback_err, msg='test')
            comp.wait_for_complete_and_cb()
Beispiel #29
0
class TestIoctx(object):

    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')

    def tearDown(self):
        self.ioctx.close()
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()

    def test_change_auid(self):
        self.ioctx.change_auid(ANONYMOUS_AUID)
        self.ioctx.change_auid(ADMIN_AUID)

    def test_write(self):
        self.ioctx.write('abc', 'abc')
        eq(self.ioctx.read('abc'), 'abc')

    def test_write_full(self):
        self.ioctx.write('abc', 'abc')
        eq(self.ioctx.read('abc'), 'abc')
        self.ioctx.write_full('abc', 'd')
        eq(self.ioctx.read('abc'), 'd')

    def test_write_zeros(self):
        self.ioctx.write('abc', 'a\0b\0c')
        eq(self.ioctx.read('abc'), 'a\0b\0c')

    def test_trunc(self):
        self.ioctx.write('abc', 'abc')
        self.ioctx.trunc('abc', 2)
        eq(self.ioctx.read('abc'), 'ab')
        size = self.ioctx.stat('abc')[0]
        eq(size, 2)

    def test_list_objects_empty(self):
        eq(list(self.ioctx.list_objects()), [])

    def test_list_objects(self):
        self.ioctx.write('a', '')
        self.ioctx.write('b', 'foo')
        self.ioctx.write_full('c', 'bar')
        object_names = [obj.key for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), ['a', 'b', 'c'])

    def test_xattrs(self):
        xattrs = dict(a='1', b='2', c='3', d='a\0b', e='\0')
        self.ioctx.write('abc', '')
        for key, value in xattrs.iteritems():
            self.ioctx.set_xattr('abc', key, value)
            eq(self.ioctx.get_xattr('abc', key), value)
        stored_xattrs = {}
        for key, value in self.ioctx.get_xattrs('abc'):
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_create_snap(self):
        assert_raises(ObjectNotFound, self.ioctx.remove_snap, 'foo')
        self.ioctx.create_snap('foo')
        self.ioctx.remove_snap('foo')

    def test_list_snaps_empty(self):
        eq(list(self.ioctx.list_snaps()), [])

    def test_list_snaps(self):
        snaps = ['snap1', 'snap2', 'snap3']
        for snap in snaps:
            self.ioctx.create_snap(snap)
        listed_snaps = [snap.name for snap in self.ioctx.list_snaps()]
        eq(snaps, listed_snaps)

    def test_lookup_snap(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        eq(snap.name, 'foo')

    def test_snap_timestamp(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        snap.get_timestamp()

    def test_remove_snap(self):
        self.ioctx.create_snap('foo')
        (snap,) = self.ioctx.list_snaps()
        eq(snap.name, 'foo')
        self.ioctx.remove_snap('foo')
        eq(list(self.ioctx.list_snaps()), [])

    def test_locator(self):
        self.ioctx.set_locator_key("bar")
        self.ioctx.write('foo', 'contents1')
        objects = [i for i in self.ioctx.list_objects()]
        eq(len(objects), 1)
        eq(self.ioctx.get_locator_key(), "bar")
        self.ioctx.set_locator_key("")
        objects[0].seek(0)
        objects[0].write("contents2")
        eq(self.ioctx.get_locator_key(), "")
        self.ioctx.set_locator_key("bar")
        contents = self.ioctx.read("foo")
        eq(contents, "contents2")
        eq(self.ioctx.get_locator_key(), "bar")
        objects[0].remove()
        objects = [i for i in self.ioctx.list_objects()]
        eq(objects, [])
        self.ioctx.set_locator_key("")

    def test_aio_write(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0
        comp = self.ioctx.aio_write("foo", "bar", 0, cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, "bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_append(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0
        comp = self.ioctx.aio_write("foo", "bar", 0, cb, cb)
        comp2 = self.ioctx.aio_append("foo", "baz", cb, cb)
        comp.wait_for_complete()
        contents = self.ioctx.read("foo")
        eq(contents, "barbaz")
        with lock:
            while count[0] < 4:
                lock.wait()
        eq(comp.get_return_value(), 0)
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_write_full(self):
        lock = threading.Condition()
        count = [0]
        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0
        self.ioctx.aio_write("foo", "barbaz", 0, cb, cb)
        comp = self.ioctx.aio_write_full("foo", "bar", cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, "bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_read(self):
        retval = [None]
        lock = threading.Condition()
        def cb(_, buf):
            with lock:
                retval[0] = buf
                lock.notify()
        self.ioctx.write("foo", "bar")
        self.ioctx.aio_read("foo", 3, 0, cb)
        with lock:
            while retval[0] is None:
                lock.wait()
        eq(retval[0], "bar")
        [i.remove() for i in self.ioctx.list_objects()]
Beispiel #30
0
class TestIoctx(object):
    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')

    def tearDown(self):
        cmd = {"prefix": "osd unset", "key": "noup"}
        self.rados.mon_command(json.dumps(cmd), b'')
        self.ioctx.close()
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()

    def test_change_auid(self):
        self.ioctx.change_auid(ANONYMOUS_AUID)
        self.ioctx.change_auid(ADMIN_AUID)

    def test_write(self):
        self.ioctx.write('abc', b'abc')
        eq(self.ioctx.read('abc'), b'abc')

    def test_write_full(self):
        self.ioctx.write('abc', b'abc')
        eq(self.ioctx.read('abc'), b'abc')
        self.ioctx.write_full('abc', b'd')
        eq(self.ioctx.read('abc'), b'd')

    def test_append(self):
        self.ioctx.write('abc', b'a')
        self.ioctx.append('abc', b'b')
        self.ioctx.append('abc', b'c')
        eq(self.ioctx.read('abc'), b'abc')

    def test_write_zeros(self):
        self.ioctx.write('abc', b'a\0b\0c')
        eq(self.ioctx.read('abc'), b'a\0b\0c')

    def test_trunc(self):
        self.ioctx.write('abc', b'abc')
        self.ioctx.trunc('abc', 2)
        eq(self.ioctx.read('abc'), b'ab')
        size = self.ioctx.stat('abc')[0]
        eq(size, 2)

    def test_list_objects_empty(self):
        eq(list(self.ioctx.list_objects()), [])

    def test_list_objects(self):
        self.ioctx.write('a', b'')
        self.ioctx.write('b', b'foo')
        self.ioctx.write_full('c', b'bar')
        self.ioctx.append('d', b'jazz')
        object_names = [obj.key for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), ['a', 'b', 'c', 'd'])

    def test_list_ns_objects(self):
        self.ioctx.write('a', b'')
        self.ioctx.write('b', b'foo')
        self.ioctx.write_full('c', b'bar')
        self.ioctx.append('d', b'jazz')
        self.ioctx.set_namespace("ns1")
        self.ioctx.write('ns1-a', b'')
        self.ioctx.write('ns1-b', b'foo')
        self.ioctx.write_full('ns1-c', b'bar')
        self.ioctx.append('ns1-d', b'jazz')
        self.ioctx.append('d', b'jazz')
        self.ioctx.set_namespace(LIBRADOS_ALL_NSPACES)
        object_names = [(obj.nspace, obj.key)
                        for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), [('', 'a'), ('','b'), ('','c'), ('','d'),\
                ('ns1', 'd'), ('ns1', 'ns1-a'), ('ns1', 'ns1-b'),\
                ('ns1', 'ns1-c'), ('ns1', 'ns1-d')])

    def test_xattrs(self):
        xattrs = dict(a=b'1', b=b'2', c=b'3', d=b'a\0b', e=b'\0')
        self.ioctx.write('abc', b'')
        for key, value in xattrs.items():
            self.ioctx.set_xattr('abc', key, value)
            eq(self.ioctx.get_xattr('abc', key), value)
        stored_xattrs = {}
        for key, value in self.ioctx.get_xattrs('abc'):
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_obj_xattrs(self):
        xattrs = dict(a=b'1', b=b'2', c=b'3', d=b'a\0b', e=b'\0')
        self.ioctx.write('abc', b'')
        obj = list(self.ioctx.list_objects())[0]
        for key, value in xattrs.items():
            obj.set_xattr(key, value)
            eq(obj.get_xattr(key), value)
        stored_xattrs = {}
        for key, value in obj.get_xattrs():
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_create_snap(self):
        assert_raises(ObjectNotFound, self.ioctx.remove_snap, 'foo')
        self.ioctx.create_snap('foo')
        self.ioctx.remove_snap('foo')

    def test_list_snaps_empty(self):
        eq(list(self.ioctx.list_snaps()), [])

    def test_list_snaps(self):
        snaps = ['snap1', 'snap2', 'snap3']
        for snap in snaps:
            self.ioctx.create_snap(snap)
        listed_snaps = [snap.name for snap in self.ioctx.list_snaps()]
        eq(snaps, listed_snaps)

    def test_lookup_snap(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        eq(snap.name, 'foo')

    def test_snap_timestamp(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        snap.get_timestamp()

    def test_remove_snap(self):
        self.ioctx.create_snap('foo')
        (snap, ) = self.ioctx.list_snaps()
        eq(snap.name, 'foo')
        self.ioctx.remove_snap('foo')
        eq(list(self.ioctx.list_snaps()), [])

    def test_snap_rollback(self):
        self.ioctx.write("insnap", b"contents1")
        self.ioctx.create_snap("snap1")
        self.ioctx.remove_object("insnap")
        self.ioctx.snap_rollback("insnap", "snap1")
        eq(self.ioctx.read("insnap"), b"contents1")
        self.ioctx.remove_snap("snap1")
        self.ioctx.remove_object("insnap")

    def test_snap_read(self):
        self.ioctx.write("insnap", b"contents1")
        self.ioctx.create_snap("snap1")
        self.ioctx.remove_object("insnap")
        snap = self.ioctx.lookup_snap("snap1")
        self.ioctx.set_read(int(snap.snap_id.value))
        eq(self.ioctx.read("insnap"), b"contents1")
        self.ioctx.set_read(LIBRADOS_SNAP_HEAD)
        self.ioctx.write("inhead", b"contents2")
        eq(self.ioctx.read("inhead"), b"contents2")
        self.ioctx.remove_snap("snap1")
        self.ioctx.remove_object("inhead")

    def test_set_omap(self):
        keys = ("1", "2", "3", "4")
        values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 4)
            self.ioctx.operate_read_op(read_op, "hw")
            iter.next()
            eq(list(iter), [("2", b"bbb"), ("3", b"ccc"),
                            ("4", b"\x04\x04\x04\x04")])

    def test_get_omap_vals_by_keys(self):
        keys = ("1", "2", "3", "4")
        values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals_by_keys(
                read_op, (
                    "3",
                    "4",
                ))
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [("3", b"ccc"), ("4", b"\x04\x04\x04\x04")])

    def test_get_omap_keys(self):
        keys = ("1", "2", "3")
        values = (b"aaa", b"bbb", b"ccc")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_keys(read_op, "", 2)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [("1", None), ("2", None)])

    def test_clear_omap(self):
        keys = ("1", "2", "3")
        values = (b"aaa", b"bbb", b"ccc")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with WriteOpCtx(self.ioctx) as write_op_1:
            self.ioctx.clear_omap(write_op_1)
            self.ioctx.operate_write_op(write_op_1, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals_by_keys(read_op, ("1", ))
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [])

    def test_locator(self):
        self.ioctx.set_locator_key("bar")
        self.ioctx.write('foo', b'contents1')
        objects = [i for i in self.ioctx.list_objects()]
        eq(len(objects), 1)
        eq(self.ioctx.get_locator_key(), "bar")
        self.ioctx.set_locator_key("")
        objects[0].seek(0)
        objects[0].write(b"contents2")
        eq(self.ioctx.get_locator_key(), "")
        self.ioctx.set_locator_key("bar")
        contents = self.ioctx.read("foo")
        eq(contents, b"contents2")
        eq(self.ioctx.get_locator_key(), "bar")
        objects[0].remove()
        objects = [i for i in self.ioctx.list_objects()]
        eq(objects, [])
        self.ioctx.set_locator_key("")

    def test_aio_write(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        comp = self.ioctx.aio_write("foo", b"bar", 0, cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, b"bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_append(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        comp = self.ioctx.aio_write("foo", b"bar", 0, cb, cb)
        comp2 = self.ioctx.aio_append("foo", b"baz", cb, cb)
        comp.wait_for_complete()
        contents = self.ioctx.read("foo")
        eq(contents, b"barbaz")
        with lock:
            while count[0] < 4:
                lock.wait()
        eq(comp.get_return_value(), 0)
        eq(comp2.get_return_value(), 0)
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_write_full(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        self.ioctx.aio_write("foo", b"barbaz", 0, cb, cb)
        comp = self.ioctx.aio_write_full("foo", b"bar", cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, b"bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def _take_down_acting_set(self, pool, objectname):
        # find acting_set for pool:objectname and take it down; used to
        # verify that async reads don't complete while acting set is missing
        cmd = {
            "prefix": "osd map",
            "pool": pool,
            "object": objectname,
            "format": "json",
        }
        r, jsonout, _ = self.rados.mon_command(json.dumps(cmd), b'')
        objmap = json.loads(jsonout.decode("utf-8"))
        acting_set = objmap['acting']
        cmd = {"prefix": "osd set", "key": "noup"}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), b'')
        eq(r, 0)
        cmd = {"prefix": "osd down", "ids": [str(i) for i in acting_set]}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), b'')
        eq(r, 0)

        # wait for OSDs to acknowledge the down
        eq(self.rados.wait_for_latest_osdmap(), 0)

    def _let_osds_back_up(self):
        cmd = {"prefix": "osd unset", "key": "noup"}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), b'')
        eq(r, 0)

    def test_aio_read(self):
        # this is a list so that the local cb() can modify it
        retval = [None]
        lock = threading.Condition()

        def cb(_, buf):
            with lock:
                retval[0] = buf
                lock.notify()

        payload = b"bar\000frob"
        self.ioctx.write("foo", payload)

        # test1: use wait_for_complete() and wait for cb by
        # watching retval[0]
        self._take_down_acting_set('test_pool', 'foo')
        comp = self.ioctx.aio_read("foo", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(None, retval[0])
        self._let_osds_back_up()
        comp.wait_for_complete()
        loops = 0
        with lock:
            while retval[0] is None and loops <= 10:
                lock.wait(timeout=5)
                loops += 1
        assert (loops <= 10)

        eq(retval[0], payload)

        # test2: use wait_for_complete_and_cb(), verify retval[0] is
        # set by the time we regain control

        retval[0] = None
        self._take_down_acting_set('test_pool', 'foo')
        comp = self.ioctx.aio_read("foo", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(None, retval[0])
        self._let_osds_back_up()

        comp.wait_for_complete_and_cb()
        assert (retval[0] is not None)
        eq(retval[0], payload)

        [i.remove() for i in self.ioctx.list_objects()]

    def test_lock(self):
        self.ioctx.lock_exclusive("foo", "lock", "locker", "desc_lock", 10000,
                                  0)
        assert_raises(ObjectExists, self.ioctx.lock_exclusive, "foo", "lock",
                      "locker", "desc_lock", 10000, 0)
        self.ioctx.unlock("foo", "lock", "locker")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock",
                      "locker")

        self.ioctx.lock_shared("foo", "lock", "locker1", "tag", "desc_lock",
                               10000, 0)
        self.ioctx.lock_shared("foo", "lock", "locker2", "tag", "desc_lock",
                               10000, 0)
        assert_raises(ObjectBusy, self.ioctx.lock_exclusive, "foo", "lock",
                      "locker3", "desc_lock", 10000, 0)
        self.ioctx.unlock("foo", "lock", "locker1")
        self.ioctx.unlock("foo", "lock", "locker2")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock",
                      "locker1")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock",
                      "locker2")

    def test_execute(self):
        self.ioctx.write("foo", "")  # ensure object exists

        ret, buf = self.ioctx.execute("foo", "hello", "say_hello", "")
        eq(buf, "Hello, world!")

        ret, buf = self.ioctx.execute("foo", "hello", "say_hello", "nose")
        eq(buf, "Hello, nose!")
Beispiel #31
0
class TestIoctx(object):
    def setUp(self):
        self.rados = Rados(conffile='')
        self.rados.connect()
        self.rados.create_pool('test_pool')
        assert self.rados.pool_exists('test_pool')
        self.ioctx = self.rados.open_ioctx('test_pool')

    def tearDown(self):
        cmd = {"prefix": "osd unset", "key": "noup"}
        self.rados.mon_command(json.dumps(cmd), b'')
        self.ioctx.close()
        self.rados.delete_pool('test_pool')
        self.rados.shutdown()

    def test_get_last_version(self):
        version = self.ioctx.get_last_version()
        assert version >= 0

    def test_get_stats(self):
        stats = self.ioctx.get_stats()
        eq(
            stats, {
                'num_objects_unfound': 0,
                'num_objects_missing_on_primary': 0,
                'num_object_clones': 0,
                'num_objects': 0,
                'num_object_copies': 0,
                'num_bytes': 0,
                'num_rd_kb': 0,
                'num_wr_kb': 0,
                'num_kb': 0,
                'num_wr': 0,
                'num_objects_degraded': 0,
                'num_rd': 0
            })

    def test_change_auid(self):
        self.ioctx.change_auid(ANONYMOUS_AUID)
        self.ioctx.change_auid(ADMIN_AUID)

    def test_write(self):
        self.ioctx.write('abc', b'abc')
        eq(self.ioctx.read('abc'), b'abc')

    def test_write_full(self):
        self.ioctx.write('abc', b'abc')
        eq(self.ioctx.read('abc'), b'abc')
        self.ioctx.write_full('abc', b'd')
        eq(self.ioctx.read('abc'), b'd')

    def test_append(self):
        self.ioctx.write('abc', b'a')
        self.ioctx.append('abc', b'b')
        self.ioctx.append('abc', b'c')
        eq(self.ioctx.read('abc'), b'abc')

    def test_write_zeros(self):
        self.ioctx.write('abc', b'a\0b\0c')
        eq(self.ioctx.read('abc'), b'a\0b\0c')

    def test_trunc(self):
        self.ioctx.write('abc', b'abc')
        self.ioctx.trunc('abc', 2)
        eq(self.ioctx.read('abc'), b'ab')
        size = self.ioctx.stat('abc')[0]
        eq(size, 2)

    def test_list_objects_empty(self):
        eq(list(self.ioctx.list_objects()), [])

    def test_list_objects(self):
        self.ioctx.write('a', b'')
        self.ioctx.write('b', b'foo')
        self.ioctx.write_full('c', b'bar')
        self.ioctx.append('d', b'jazz')
        object_names = [obj.key for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), ['a', 'b', 'c', 'd'])

    def test_list_ns_objects(self):
        self.ioctx.write('a', b'')
        self.ioctx.write('b', b'foo')
        self.ioctx.write_full('c', b'bar')
        self.ioctx.append('d', b'jazz')
        self.ioctx.set_namespace("ns1")
        self.ioctx.write('ns1-a', b'')
        self.ioctx.write('ns1-b', b'foo')
        self.ioctx.write_full('ns1-c', b'bar')
        self.ioctx.append('ns1-d', b'jazz')
        self.ioctx.append('d', b'jazz')
        self.ioctx.set_namespace(LIBRADOS_ALL_NSPACES)
        object_names = [(obj.nspace, obj.key)
                        for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), [('', 'a'), ('','b'), ('','c'), ('','d'),\
                ('ns1', 'd'), ('ns1', 'ns1-a'), ('ns1', 'ns1-b'),\
                ('ns1', 'ns1-c'), ('ns1', 'ns1-d')])

    def test_xattrs(self):
        xattrs = dict(a=b'1', b=b'2', c=b'3', d=b'a\0b', e=b'\0')
        self.ioctx.write('abc', b'')
        for key, value in xattrs.items():
            self.ioctx.set_xattr('abc', key, value)
            eq(self.ioctx.get_xattr('abc', key), value)
        stored_xattrs = {}
        for key, value in self.ioctx.get_xattrs('abc'):
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_obj_xattrs(self):
        xattrs = dict(a=b'1', b=b'2', c=b'3', d=b'a\0b', e=b'\0')
        self.ioctx.write('abc', b'')
        obj = list(self.ioctx.list_objects())[0]
        for key, value in xattrs.items():
            obj.set_xattr(key, value)
            eq(obj.get_xattr(key), value)
        stored_xattrs = {}
        for key, value in obj.get_xattrs():
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_create_snap(self):
        assert_raises(ObjectNotFound, self.ioctx.remove_snap, 'foo')
        self.ioctx.create_snap('foo')
        self.ioctx.remove_snap('foo')

    def test_list_snaps_empty(self):
        eq(list(self.ioctx.list_snaps()), [])

    def test_list_snaps(self):
        snaps = ['snap1', 'snap2', 'snap3']
        for snap in snaps:
            self.ioctx.create_snap(snap)
        listed_snaps = [snap.name for snap in self.ioctx.list_snaps()]
        eq(snaps, listed_snaps)

    def test_lookup_snap(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        eq(snap.name, 'foo')

    def test_snap_timestamp(self):
        self.ioctx.create_snap('foo')
        snap = self.ioctx.lookup_snap('foo')
        snap.get_timestamp()

    def test_remove_snap(self):
        self.ioctx.create_snap('foo')
        (snap, ) = self.ioctx.list_snaps()
        eq(snap.name, 'foo')
        self.ioctx.remove_snap('foo')
        eq(list(self.ioctx.list_snaps()), [])

    def test_snap_rollback(self):
        self.ioctx.write("insnap", b"contents1")
        self.ioctx.create_snap("snap1")
        self.ioctx.remove_object("insnap")
        self.ioctx.snap_rollback("insnap", "snap1")
        eq(self.ioctx.read("insnap"), b"contents1")
        self.ioctx.remove_snap("snap1")
        self.ioctx.remove_object("insnap")

    def test_snap_read(self):
        self.ioctx.write("insnap", b"contents1")
        self.ioctx.create_snap("snap1")
        self.ioctx.remove_object("insnap")
        snap = self.ioctx.lookup_snap("snap1")
        self.ioctx.set_read(snap.snap_id)
        eq(self.ioctx.read("insnap"), b"contents1")
        self.ioctx.set_read(LIBRADOS_SNAP_HEAD)
        self.ioctx.write("inhead", b"contents2")
        eq(self.ioctx.read("inhead"), b"contents2")
        self.ioctx.remove_snap("snap1")
        self.ioctx.remove_object("inhead")

    def test_set_omap(self):
        keys = ("1", "2", "3", "4")
        values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            write_op.set_flags(LIBRADOS_OPERATION_SKIPRWLOCKS)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 4)
            eq(ret, 0)
            self.ioctx.operate_read_op(read_op, "hw")
            next(iter)
            eq(list(iter), [("2", b"bbb"), ("3", b"ccc"),
                            ("4", b"\x04\x04\x04\x04")])
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals(read_op, "2", "", 4)
            eq(ret, 0)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(("3", b"ccc"), next(iter))
            eq(list(iter), [("4", b"\x04\x04\x04\x04")])
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals(read_op, "", "2", 4)
            eq(ret, 0)
            read_op.set_flags(LIBRADOS_OPERATION_BALANCE_READS)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [("2", b"bbb")])

    def test_set_omap_aio(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        keys = ("1", "2", "3", "4")
        values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            comp = self.ioctx.operate_aio_write_op(write_op, "hw", cb, cb)
            comp.wait_for_complete()
            comp.wait_for_safe()
            with lock:
                while count[0] < 2:
                    lock.wait()
            eq(comp.get_return_value(), 0)

        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 4)
            eq(ret, 0)
            comp = self.ioctx.operate_aio_read_op(read_op, "hw", cb, cb)
            comp.wait_for_complete()
            comp.wait_for_safe()
            with lock:
                while count[0] < 4:
                    lock.wait()
            eq(comp.get_return_value(), 0)
            next(iter)
            eq(list(iter), [("2", b"bbb"), ("3", b"ccc"),
                            ("4", b"\x04\x04\x04\x04")])

    def test_write_ops(self):
        with WriteOpCtx(self.ioctx) as write_op:
            write_op.new(0)
            self.ioctx.operate_write_op(write_op, "write_ops")
            eq(self.ioctx.read('write_ops'), b'')
            write_op.write_full(b'1')
            write_op.append(b'2')
            self.ioctx.operate_write_op(write_op, "write_ops")
            eq(self.ioctx.read('write_ops'), b'12')
            write_op.write_full(b'12345')
            write_op.write(b'x', 2)
            self.ioctx.operate_write_op(write_op, "write_ops")
            eq(self.ioctx.read('write_ops'), b'12x45')
            write_op.write_full(b'12345')
            write_op.zero(2, 2)
            self.ioctx.operate_write_op(write_op, "write_ops")
            eq(self.ioctx.read('write_ops'), b'12\x00\x005')
            write_op.write_full(b'12345')
            write_op.truncate(2)
            self.ioctx.operate_write_op(write_op, "write_ops")
            eq(self.ioctx.read('write_ops'), b'12')
            write_op.remove()
            self.ioctx.operate_write_op(write_op, "write_ops")
            with assert_raises(ObjectNotFound):
                self.ioctx.read('write_ops')

    def test_get_omap_vals_by_keys(self):
        keys = ("1", "2", "3", "4")
        values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals_by_keys(
                read_op, (
                    "3",
                    "4",
                ))
            eq(ret, 0)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [("3", b"ccc"), ("4", b"\x04\x04\x04\x04")])
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals_by_keys(
                read_op, (
                    "3",
                    "4",
                ))
            eq(ret, 0)
            with assert_raises(ObjectNotFound):
                self.ioctx.operate_read_op(read_op, "no_such")

    def test_get_omap_keys(self):
        keys = ("1", "2", "3")
        values = (b"aaa", b"bbb", b"ccc")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_keys(read_op, "", 2)
            eq(ret, 0)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [("1", None), ("2", None)])
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_keys(read_op, "", 2)
            eq(ret, 0)
            with assert_raises(ObjectNotFound):
                self.ioctx.operate_read_op(read_op, "no_such")

    def test_clear_omap(self):
        keys = ("1", "2", "3")
        values = (b"aaa", b"bbb", b"ccc")
        with WriteOpCtx(self.ioctx) as write_op:
            self.ioctx.set_omap(write_op, keys, values)
            self.ioctx.operate_write_op(write_op, "hw")
        with WriteOpCtx(self.ioctx) as write_op_1:
            self.ioctx.clear_omap(write_op_1)
            self.ioctx.operate_write_op(write_op_1, "hw")
        with ReadOpCtx(self.ioctx) as read_op:
            iter, ret = self.ioctx.get_omap_vals_by_keys(read_op, ("1", ))
            eq(ret, 0)
            self.ioctx.operate_read_op(read_op, "hw")
            eq(list(iter), [])

    def test_locator(self):
        self.ioctx.set_locator_key("bar")
        self.ioctx.write('foo', b'contents1')
        objects = [i for i in self.ioctx.list_objects()]
        eq(len(objects), 1)
        eq(self.ioctx.get_locator_key(), "bar")
        self.ioctx.set_locator_key("")
        objects[0].seek(0)
        objects[0].write(b"contents2")
        eq(self.ioctx.get_locator_key(), "")
        self.ioctx.set_locator_key("bar")
        contents = self.ioctx.read("foo")
        eq(contents, b"contents2")
        eq(self.ioctx.get_locator_key(), "bar")
        objects[0].remove()
        objects = [i for i in self.ioctx.list_objects()]
        eq(objects, [])
        self.ioctx.set_locator_key("")

    def test_aio_write(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        comp = self.ioctx.aio_write("foo", b"bar", 0, cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, b"bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_write_no_comp_ref(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        # NOTE(sileht): force don't save the comp into local var
        # to ensure all references are correctly tracked into the lib
        self.ioctx.aio_write("foo", b"bar", 0, cb, cb)
        with lock:
            while count[0] < 2:
                lock.wait()
        contents = self.ioctx.read("foo")
        eq(contents, b"bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_append(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        comp = self.ioctx.aio_write("foo", b"bar", 0, cb, cb)
        comp2 = self.ioctx.aio_append("foo", b"baz", cb, cb)
        comp.wait_for_complete()
        contents = self.ioctx.read("foo")
        eq(contents, b"barbaz")
        with lock:
            while count[0] < 4:
                lock.wait()
        eq(comp.get_return_value(), 0)
        eq(comp2.get_return_value(), 0)
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_write_full(self):
        lock = threading.Condition()
        count = [0]

        def cb(blah):
            with lock:
                count[0] += 1
                lock.notify()
            return 0

        self.ioctx.aio_write("foo", b"barbaz", 0, cb, cb)
        comp = self.ioctx.aio_write_full("foo", b"bar", cb, cb)
        comp.wait_for_complete()
        comp.wait_for_safe()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)
        contents = self.ioctx.read("foo")
        eq(contents, b"bar")
        [i.remove() for i in self.ioctx.list_objects()]

    def test_aio_stat(self):
        lock = threading.Condition()
        count = [0]

        def cb(_, size, mtime):
            with lock:
                count[0] += 1
                lock.notify()

        comp = self.ioctx.aio_stat("foo", cb)
        comp.wait_for_complete()
        with lock:
            while count[0] < 1:
                lock.wait()
        eq(comp.get_return_value(), -2)

        self.ioctx.write("foo", b"bar")

        comp = self.ioctx.aio_stat("foo", cb)
        comp.wait_for_complete()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 0)

        [i.remove() for i in self.ioctx.list_objects()]

    def _take_down_acting_set(self, pool, objectname):
        # find acting_set for pool:objectname and take it down; used to
        # verify that async reads don't complete while acting set is missing
        cmd = {
            "prefix": "osd map",
            "pool": pool,
            "object": objectname,
            "format": "json",
        }
        r, jsonout, _ = self.rados.mon_command(json.dumps(cmd), b'')
        objmap = json.loads(jsonout.decode("utf-8"))
        acting_set = objmap['acting']
        cmd = {"prefix": "osd set", "key": "noup"}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), b'')
        eq(r, 0)
        cmd = {"prefix": "osd down", "ids": [str(i) for i in acting_set]}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), b'')
        eq(r, 0)

        # wait for OSDs to acknowledge the down
        eq(self.rados.wait_for_latest_osdmap(), 0)

    def _let_osds_back_up(self):
        cmd = {"prefix": "osd unset", "key": "noup"}
        r, _, _ = self.rados.mon_command(json.dumps(cmd), b'')
        eq(r, 0)

    def test_aio_read(self):
        # this is a list so that the local cb() can modify it
        retval = [None]
        lock = threading.Condition()

        def cb(_, buf):
            with lock:
                retval[0] = buf
                lock.notify()

        payload = b"bar\000frob"
        self.ioctx.write("foo", payload)

        # test1: use wait_for_complete() and wait for cb by
        # watching retval[0]
        self._take_down_acting_set('test_pool', 'foo')
        comp = self.ioctx.aio_read("foo", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(None, retval[0])
        self._let_osds_back_up()
        comp.wait_for_complete()
        loops = 0
        with lock:
            while retval[0] is None and loops <= 10:
                lock.wait(timeout=5)
                loops += 1
        assert (loops <= 10)

        eq(retval[0], payload)
        eq(sys.getrefcount(comp), 2)

        # test2: use wait_for_complete_and_cb(), verify retval[0] is
        # set by the time we regain control

        retval[0] = None
        self._take_down_acting_set('test_pool', 'foo')
        comp = self.ioctx.aio_read("foo", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(None, retval[0])
        self._let_osds_back_up()

        comp.wait_for_complete_and_cb()
        assert (retval[0] is not None)
        eq(retval[0], payload)
        eq(sys.getrefcount(comp), 2)

        # test3: error case, use wait_for_complete_and_cb(), verify retval[0] is
        # set by the time we regain control

        retval[0] = 1
        self._take_down_acting_set('test_pool', 'bar')
        comp = self.ioctx.aio_read("bar", len(payload), 0, cb)
        eq(False, comp.is_complete())
        time.sleep(3)
        eq(False, comp.is_complete())
        with lock:
            eq(1, retval[0])
        self._let_osds_back_up()

        comp.wait_for_complete_and_cb()
        eq(None, retval[0])
        assert (comp.get_return_value() < 0)
        eq(sys.getrefcount(comp), 2)

        [i.remove() for i in self.ioctx.list_objects()]

    def test_lock(self):
        self.ioctx.lock_exclusive("foo", "lock", "locker", "desc_lock", 10000,
                                  0)
        assert_raises(ObjectExists, self.ioctx.lock_exclusive, "foo", "lock",
                      "locker", "desc_lock", 10000, 0)
        self.ioctx.unlock("foo", "lock", "locker")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock",
                      "locker")

        self.ioctx.lock_shared("foo", "lock", "locker1", "tag", "desc_lock",
                               10000, 0)
        self.ioctx.lock_shared("foo", "lock", "locker2", "tag", "desc_lock",
                               10000, 0)
        assert_raises(ObjectBusy, self.ioctx.lock_exclusive, "foo", "lock",
                      "locker3", "desc_lock", 10000, 0)
        self.ioctx.unlock("foo", "lock", "locker1")
        self.ioctx.unlock("foo", "lock", "locker2")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock",
                      "locker1")
        assert_raises(ObjectNotFound, self.ioctx.unlock, "foo", "lock",
                      "locker2")

    def test_execute(self):
        self.ioctx.write("foo", b"")  # ensure object exists

        ret, buf = self.ioctx.execute("foo", "hello", "say_hello", b"")
        eq(buf, b"Hello, world!")

        ret, buf = self.ioctx.execute("foo", "hello", "say_hello", b"nose")
        eq(buf, b"Hello, nose!")

    def test_aio_execute(self):
        count = [0]
        retval = [None]
        lock = threading.Condition()

        def cb(_, buf):
            with lock:
                if retval[0] is None:
                    retval[0] = buf
                count[0] += 1
                lock.notify()

        self.ioctx.write("foo", b"")  # ensure object exists

        comp = self.ioctx.aio_execute("foo", "hello", "say_hello", b"", 32, cb,
                                      cb)
        comp.wait_for_complete()
        with lock:
            while count[0] < 2:
                lock.wait()
        eq(comp.get_return_value(), 13)
        eq(retval[0], b"Hello, world!")

        retval[0] = None
        comp = self.ioctx.aio_execute("foo", "hello", "say_hello", b"nose", 32,
                                      cb, cb)
        comp.wait_for_complete()
        with lock:
            while count[0] < 4:
                lock.wait()
        eq(comp.get_return_value(), 12)
        eq(retval[0], b"Hello, nose!")

        [i.remove() for i in self.ioctx.list_objects()]

    def test_applications(self):
        cmd = {"prefix": "osd dump", "format": "json"}
        ret, buf, errs = self.rados.mon_command(json.dumps(cmd), b'')
        eq(ret, 0)
        assert len(buf) > 0
        release = json.loads(buf.decode("utf-8")).get("require_osd_release",
                                                      None)
        if not release or release[0] < 'l':
            raise SkipTest

        eq([], self.ioctx.application_list())

        self.ioctx.application_enable("app1")
        assert_raises(Error, self.ioctx.application_enable, "app2")
        self.ioctx.application_enable("app2", True)

        assert_raises(Error, self.ioctx.application_metadata_list, "dne")
        eq([], self.ioctx.application_metadata_list("app1"))

        assert_raises(Error, self.ioctx.application_metadata_set, "dne", "key",
                      "key")
        self.ioctx.application_metadata_set("app1", "key1", "val1")
        self.ioctx.application_metadata_set("app1", "key2", "val2")
        self.ioctx.application_metadata_set("app2", "key1", "val1")

        eq([("key1", "val1"), ("key2", "val2")],
           self.ioctx.application_metadata_list("app1"))

        self.ioctx.application_metadata_remove("app1", "key1")
        eq([("key2", "val2")], self.ioctx.application_metadata_list("app1"))
Beispiel #32
0
class TestIoctx(object):
    def setUp(self):
        self.rados = Rados(conffile="")
        self.rados.connect()
        self.rados.create_pool("test_pool")
        assert self.rados.pool_exists("test_pool")
        self.ioctx = self.rados.open_ioctx("test_pool")

    def tearDown(self):
        self.ioctx.close()
        self.rados.delete_pool("test_pool")
        self.rados.shutdown()

    def test_change_auid(self):
        self.ioctx.change_auid(ANONYMOUS_AUID)
        self.ioctx.change_auid(ADMIN_AUID)

    def test_write(self):
        self.ioctx.write("abc", "abc")
        eq(self.ioctx.read("abc"), "abc")

    def test_write_full(self):
        self.ioctx.write("abc", "abc")
        eq(self.ioctx.read("abc"), "abc")
        self.ioctx.write_full("abc", "d")
        eq(self.ioctx.read("abc"), "d")

    def test_write_zeros(self):
        self.ioctx.write("abc", "a\0b\0c")
        eq(self.ioctx.read("abc"), "a\0b\0c")

    def test_list_objects_empty(self):
        eq(list(self.ioctx.list_objects()), [])

    def test_list_objects(self):
        self.ioctx.write("a", "")
        self.ioctx.write("b", "foo")
        self.ioctx.write_full("c", "bar")
        object_names = [obj.key for obj in self.ioctx.list_objects()]
        eq(sorted(object_names), ["a", "b", "c"])

    def test_xattrs(self):
        xattrs = dict(a="1", b="2", c="3", d="a\0b", e="\0")
        self.ioctx.write("abc", "")
        for key, value in xattrs.iteritems():
            self.ioctx.set_xattr("abc", key, value)
            eq(self.ioctx.get_xattr("abc", key), value)
        stored_xattrs = {}
        for key, value in self.ioctx.get_xattrs("abc"):
            stored_xattrs[key] = value
        eq(stored_xattrs, xattrs)

    def test_create_snap(self):
        assert_raises(ObjectNotFound, self.ioctx.remove_snap, "foo")
        self.ioctx.create_snap("foo")
        self.ioctx.remove_snap("foo")

    def test_list_snaps_empty(self):
        eq(list(self.ioctx.list_snaps()), [])

    def test_list_snaps(self):
        snaps = ["snap1", "snap2", "snap3"]
        for snap in snaps:
            self.ioctx.create_snap(snap)
        listed_snaps = [snap.name for snap in self.ioctx.list_snaps()]
        eq(snaps, listed_snaps)

    def test_lookup_snap(self):
        self.ioctx.create_snap("foo")
        snap = self.ioctx.lookup_snap("foo")
        eq(snap.name, "foo")

    def test_snap_timestamp(self):
        self.ioctx.create_snap("foo")
        snap = self.ioctx.lookup_snap("foo")
        snap.get_timestamp()

    def test_remove_snap(self):
        self.ioctx.create_snap("foo")
        (snap,) = self.ioctx.list_snaps()
        eq(snap.name, "foo")
        self.ioctx.remove_snap("foo")
        eq(list(self.ioctx.list_snaps()), [])