Beispiel #1
0
class ObjectDataValidation(TestWithServers):
    """
    Test Class Description:
        Tests that create Different length records,
        Disconnect the pool/container and reconnect,
        validate the data after reconnect.

    :avocado: recursive
    """

    # pylint: disable=too-many-instance-attributes
    def setUp(self):
        super(ObjectDataValidation, self).setUp()
        self.obj = None
        self.ioreq = None
        self.no_of_dkeys = None
        self.no_of_akeys = None
        self.array_size = None
        self.record_length = None
        self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0]
        self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0]
        self.array_size = self.params.get("size", '/array_size/')
        self.record_length = self.params.get("length", '/run/record/*')

        self.prepare_pool()

        self.container = DaosContainer(self.context)
        self.container.create(self.pool.pool.handle)
        self.container.open()

        self.obj = DaosObj(self.context, self.container)
        self.obj.create(objcls=1)
        self.obj.open()
        self.ioreq = IORequest(self.context,
                               self.container,
                               self.obj,
                               objtype=4)

    def reconnect(self):
        '''
        Function to reconnect the pool/container and reopen the Object
        for read verification.
        '''
        #Close the Obj/Container, Disconnect the Pool.
        self.obj.close()
        self.container.close()
        self.pool.disconnect()
        time.sleep(5)
        #Connect Pool, Open Container and Object
        self.pool.connect(2)
        self.container.open()
        self.obj.open()
        self.ioreq = IORequest(self.context,
                               self.container,
                               self.obj,
                               objtype=4)

    @avocado.fail_on(DaosApiError)
    def test_invalid_tx_commit_close(self):
        """
        Test ID:
            (1)DAOS-1346: Verify commit tx bad parameter behavior.
            (2)DAOS-1343: Verify tx_close bad parameter behavior.
            (3)DAOS-1342: Verify tx_close through daos_api.
            (4)DAOS-1338: Add and verify tx_abort through daos_api.
            (5)DAOS-1339: Verify tx_abort bad parameter behavior.
        Test Description:
            Write Avocado Test to verify commit tx and close tx
                          bad parameter behavior.
        :avocado: tags=all,object,full_regression,small,invalid_tx
        """
        self.d_log.info("==Writing the Single Dataset for negative test...")
        record_index = 0
        expected_error = "RC: -1002"
        dkey = 0
        akey = 0
        indata = ("{0}".format(str(akey)[0]) *
                  self.record_length[record_index])
        c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
        c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
        c_value = ctypes.create_string_buffer(indata)
        c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
        try:
            new_transaction = self.container.get_new_tx()
        except DaosApiError as excep:
            #initial container get_new_tx failed, skip rest of the test
            self.fail("##container get_new_tx failed: {}".format(excep))
        invalid_transaction = new_transaction + random.randint(1000, 383838)
        self.log.info("==new_transaction=     %s", new_transaction)
        self.log.info("==invalid_transaction= %s", invalid_transaction)
        self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
                                 new_transaction)
        try:
            self.container.commit_tx(invalid_transaction)
            self.fail("##(1.1)Container.commit_tx passing with invalid handle")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.log.info(
                "==(1)Expecting failure: invalid Container.commit_tx.")
            if expected_error not in str(excep):
                self.fail(
                    "##(1.2)Expecting error RC: -1002, but got {}.".format(
                        str(excep)))
        try:
            self.container.close_tx(invalid_transaction)
            self.fail("##(2.1)Container.close_tx passing with invalid handle")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.log.info(
                "==(2)Expecting failure: invalid Container.commit_tx.")
            if expected_error not in str(excep):
                self.fail(
                    "##(2.2)Expecting error RC: -1002, but got {}.".format(
                        str(excep)))
        try:
            self.container.close_tx(new_transaction)
            self.log.info("==(3)container.close_tx test passed.")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.fail("##(3)Failed on close_tx.")

        try:
            self.container.abort_tx(invalid_transaction)
            self.fail("##(4.1)Container.abort_tx passing with invalid handle")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.log.info(
                "==(4)Expecting failure: invalid Container.abort_tx.")
            if expected_error not in str(excep):
                self.fail(
                    "##(4.2)Expecting error RC: -1002, but got {}.".format(
                        str(excep)))

        #Try to abort the transaction which already closed.
        try:
            self.container.abort_tx(new_transaction)
            self.fail("##(5.1)Container.abort_tx passing with a closed handle")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.log.info(
                "==(5)Expecting failure: Container.abort_tx closed handle.")
            if expected_error not in str(excep):
                self.fail(
                    "##(5.2)Expecting error RC: -1002, but got {}.".format(
                        str(excep)))

        #open another transaction for abort test
        try:
            new_transaction2 = self.container.get_new_tx()
        except DaosApiError as excep:
            self.fail("##(6.1)container get_new_tx failed: {}".format(excep))
        self.log.info("==new_transaction2=     %s", new_transaction2)
        self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
                                 new_transaction2)
        try:
            self.container.abort_tx(new_transaction2)
            self.log.info("==(6)container.abort_tx test passed.")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.fail("##(6.2)Failed on abort_tx.")

        self.container.close_tx(new_transaction2)

    @avocado.fail_on(DaosApiError)
    def test_single_object_validation(self):
        """
        Test ID: DAOS-707
        Test Description: Write Avocado Test to verify single data after
                          pool/container disconnect/reconnect.
        :avocado: tags=all,object,full_regression,small,single_object
        """
        self.d_log.info("Writing the Single Dataset")
        record_index = 0
        for dkey in range(self.no_of_dkeys):
            for akey in range(self.no_of_akeys):
                indata = ("{0}".format(str(akey)[0]) *
                          self.record_length[record_index])
                c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
                c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
                c_value = ctypes.create_string_buffer(indata)
                c_size = ctypes.c_size_t(ctypes.sizeof(c_value))

                self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size)
                record_index = record_index + 1
                if record_index == len(self.record_length):
                    record_index = 0

        self.reconnect()

        self.d_log.info("Single Dataset Verification -- Started")
        record_index = 0
        transaction_index = 0
        for dkey in range(self.no_of_dkeys):
            for akey in range(self.no_of_akeys):
                indata = ("{0}".format(str(akey)[0]) *
                          self.record_length[record_index])
                c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
                c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
                val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1)
                if indata != (repr(val.value)[1:-1]):
                    self.d_log.error("ERROR:Data mismatch for "
                                     "dkey = {0}, "
                                     "akey = {1}".format(
                                         "dkey {0}".format(dkey),
                                         "akey {0}".format(akey)))
                    self.fail(
                        "ERROR: Data mismatch for dkey = {0}, akey={1}".format(
                            "dkey {0}".format(dkey), "akey {0}".format(akey)))

                transaction_index = transaction_index + 1
                record_index = record_index + 1
                if record_index == len(self.record_length):
                    record_index = 0

    @avocado.fail_on(DaosApiError)
    def test_array_object_validation(self):
        """
        Test ID: DAOS-707
        Test Description: Write Avocado Test to verify Array data after
                          pool/container disconnect/reconnect.
        :avocado: tags=all,object,full_regression,small,array_object
        """
        self.d_log.info("Writing the Array Dataset")
        record_index = 0
        for dkey in range(self.no_of_dkeys):
            for akey in range(self.no_of_akeys):
                c_values = []
                value = ("{0}".format(str(akey)[0]) *
                         self.record_length[record_index])
                for item in range(self.array_size):
                    c_values.append(
                        (ctypes.create_string_buffer(value), len(value) + 1))
                c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
                c_akey = ctypes.create_string_buffer("akey {0}".format(akey))

                self.ioreq.insert_array(c_dkey, c_akey, c_values)

                record_index = record_index + 1
                if record_index == len(self.record_length):
                    record_index = 0

        self.reconnect()

        self.d_log.info("Array Dataset Verification -- Started")
        record_index = 0
        transaction_index = 0
        for dkey in range(self.no_of_dkeys):
            for akey in range(self.no_of_akeys):
                indata = []
                value = ("{0}".format(str(akey)[0]) *
                         self.record_length[record_index])
                for item in range(self.array_size):
                    indata.append(value)
                c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
                c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
                c_rec_count = ctypes.c_uint(len(indata))
                c_rec_size = ctypes.c_size_t(len(indata[0]) + 1)

                outdata = self.ioreq.fetch_array(c_dkey, c_akey, c_rec_count,
                                                 c_rec_size)

                for item in enumerate(indata):
                    if indata[item[0]] != outdata[item[0]][:-1]:
                        self.d_log.error("ERROR:Data mismatch for "
                                         "dkey = {0}, "
                                         "akey = {1}".format(
                                             "dkey {0}".format(dkey),
                                             "akey {0}".format(akey)))
                        self.fail(
                            "ERROR:Data mismatch for dkey = {0}, akey={1}".
                            format("dkey {0}".format(dkey),
                                   "akey {0}".format(akey)))

                transaction_index = transaction_index + 1
                record_index = record_index + 1
                if record_index == len(self.record_length):
                    record_index = 0
class ObjectDataValidation(avocado.Test):
    """
    Test Class Description:
        Tests that create Different length records,
        Disconnect the pool/container and reconnect,
        validate the data after reconnect.
    """

    # pylint: disable=too-many-instance-attributes
    def setUp(self):
        self.agent_sessions = None
        self.pool = None
        self.container = None
        self.obj = None
        self.ioreq = None
        self.hostlist = None
        self.hostfile = None
        self.no_of_dkeys = None
        self.no_of_akeys = None
        self.array_size = None
        self.record_length = None

        with open('../../.build_vars.json') as json_f:
            build_paths = json.load(json_f)
        self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../")
        server_group = self.params.get("name", '/server_config/',
                                       'daos_server')
        self.context = DaosContext(build_paths['PREFIX'] + '/lib64/')
        self.d_log = DaosLog(self.context)
        self.hostlist = self.params.get("test_machines", '/run/hosts/*')
        self.hostfile = write_host_file.write_host_file(
            self.hostlist, self.workdir)
        self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0]
        self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0]
        self.array_size = self.params.get("size", '/array_size/')
        self.record_length = self.params.get("length", '/run/record/*')

        self.agent_sessions = agent_utils.run_agent(self.basepath,
                                                    self.hostlist)
        server_utils.run_server(self, self.hostfile, server_group)

        self.pool = DaosPool(self.context)
        self.pool.create(self.params.get("mode", '/run/pool/createmode/*'),
                         os.geteuid(), os.getegid(),
                         self.params.get("size", '/run/pool/createsize/*'),
                         self.params.get("setname", '/run/pool/createset/*'),
                         None)
        self.pool.connect(2)

        self.container = DaosContainer(self.context)
        self.container.create(self.pool.handle)
        self.container.open()

        self.obj = DaosObj(self.context, self.container)
        self.obj.create(objcls=1)
        self.obj.open()
        self.ioreq = IORequest(self.context,
                               self.container,
                               self.obj,
                               objtype=4)

    def tearDown(self):
        try:
            if self.container:
                self.container.close()
                self.container.destroy()
            if self.pool:
                self.pool.disconnect()
                self.pool.destroy(1)
        finally:
            if self.agent_sessions:
                agent_utils.stop_agent(self.agent_sessions)
            server_utils.stop_server(hosts=self.hostlist)

    def reconnect(self):
        '''
        Function to reconnect the pool/container and reopen the Object
        for read verification.
        '''
        #Close the Obj/Container, Disconnect the Pool.
        self.obj.close()
        self.container.close()
        self.pool.disconnect()
        time.sleep(5)
        #Connect Pool, Open Container and Object
        self.pool.connect(2)
        self.container.open()
        self.obj.open()
        self.ioreq = IORequest(self.context,
                               self.container,
                               self.obj,
                               objtype=4)

    @avocado.fail_on(DaosApiError)
    def test_invalid_tx_commit_close(self):
        """
        Test ID:
            (1)DAOS-1346: Verify commit tx bad parameter behavior.
            (2)DAOS-1343: Verify tx_close bad parameter behavior.
            (3)DAOS-1342: Verify tx_close through daos_api.
            (4)DAOS-1338: Add and verify tx_abort through daos_api.
            (5)DAOS-1339: Verify tx_abort bad parameter behavior.
        Test Description:
            Write Avocado Test to verify commit tx and close tx
                          bad parameter behavior.
        :avocado: tags=all,object,full_regression,small,invalid_tx
        """
        self.d_log.info("==Writing the Single Dataset for negative test...")
        record_index = 0
        expected_error = "RC: -1002"
        dkey = 0
        akey = 0
        indata = ("{0}".format(str(akey)[0]) *
                  self.record_length[record_index])
        c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
        c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
        c_value = ctypes.create_string_buffer(indata)
        c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
        try:
            new_transaction = self.container.get_new_tx()
        except DaosApiError as excep:
            #initial container get_new_tx failed, skip rest of the test
            self.fail("##container get_new_tx failed: {}".format(excep))
        invalid_transaction = new_transaction + random.randint(1000, 383838)
        self.log.info("==new_transaction=     %s", new_transaction)
        self.log.info("==invalid_transaction= %s", invalid_transaction)
        self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
                                 new_transaction)
        try:
            self.container.commit_tx(invalid_transaction)
            self.fail("##(1.1)Container.commit_tx passing with invalid handle")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.log.info(
                "==(1)Expecting failure: invalid Container.commit_tx.")
            if expected_error not in str(excep):
                self.fail(
                    "##(1.2)Expecting error RC: -1002, but got {}.".format(
                        str(excep)))
        try:
            self.container.close_tx(invalid_transaction)
            self.fail("##(2.1)Container.close_tx passing with invalid handle")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.log.info(
                "==(2)Expecting failure: invalid Container.commit_tx.")
            if expected_error not in str(excep):
                self.fail(
                    "##(2.2)Expecting error RC: -1002, but got {}.".format(
                        str(excep)))
        try:
            self.container.close_tx(new_transaction)
            self.log.info("==(3)container.close_tx test passed.")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.fail("##(3)Failed on close_tx.")

        try:
            self.container.abort_tx(invalid_transaction)
            self.fail("##(4.1)Container.abort_tx passing with invalid handle")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.log.info(
                "==(4)Expecting failure: invalid Container.abort_tx.")
            if expected_error not in str(excep):
                self.fail(
                    "##(4.2)Expecting error RC: -1002, but got {}.".format(
                        str(excep)))

        #Try to abort the transaction which already closed.
        try:
            self.container.abort_tx(new_transaction)
            self.fail("##(5.1)Container.abort_tx passing with a closed handle")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.log.info(
                "==(5)Expecting failure: Container.abort_tx closed handle.")
            if expected_error not in str(excep):
                self.fail(
                    "##(5.2)Expecting error RC: -1002, but got {}.".format(
                        str(excep)))

        #open another transaction for abort test
        try:
            new_transaction2 = self.container.get_new_tx()
        except DaosApiError as excep:
            self.fail("##(6.1)container get_new_tx failed: {}".format(excep))
        self.log.info("==new_transaction2=     %s", new_transaction2)
        self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
                                 new_transaction2)
        try:
            self.container.abort_tx(new_transaction2)
            self.log.info("==(6)container.abort_tx test passed.")
        except DaosApiError as excep:
            self.log.info(str(excep))
            self.fail("##(6.2)Failed on abort_tx.")

    @avocado.fail_on(DaosApiError)
    @skipForTicket("DAOS-3208")
    def test_single_object_validation(self):
        """
        Test ID: DAOS-707
        Test Description: Write Avocado Test to verify single data after
                          pool/container disconnect/reconnect.
        :avocado: tags=all,object,full_regression,small,single_object
        """
        self.d_log.info("Writing the Single Dataset")
        record_index = 0
        transaction = []
        for dkey in range(self.no_of_dkeys):
            for akey in range(self.no_of_akeys):
                indata = ("{0}".format(str(akey)[0]) *
                          self.record_length[record_index])
                c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
                c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
                c_value = ctypes.create_string_buffer(indata)
                c_size = ctypes.c_size_t(ctypes.sizeof(c_value))

                new_transaction = self.container.get_new_tx()
                self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
                                         new_transaction)
                self.container.commit_tx(new_transaction)
                transaction.append(new_transaction)
                record_index = record_index + 1
                if record_index == len(self.record_length):
                    record_index = 0

        self.reconnect()

        self.d_log.info("Single Dataset Verification -- Started")
        record_index = 0
        transaction_index = 0
        for dkey in range(self.no_of_dkeys):
            for akey in range(self.no_of_akeys):
                indata = ("{0}".format(str(akey)[0]) *
                          self.record_length[record_index])
                c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
                c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
                val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1)
                if indata != (repr(val.value)[1:-1]):
                    self.d_log.error("ERROR:Data mismatch for "
                                     "dkey = {0}, "
                                     "akey = {1}".format(
                                         "dkey {0}".format(dkey),
                                         "akey {0}".format(akey)))
                    self.fail(
                        "ERROR: Data mismatch for dkey = {0}, akey={1}".format(
                            "dkey {0}".format(dkey), "akey {0}".format(akey)))

                transaction_index = transaction_index + 1
                record_index = record_index + 1
                if record_index == len(self.record_length):
                    record_index = 0

    @avocado.fail_on(DaosApiError)
    @skipForTicket("DAOS-3208")
    def test_array_object_validation(self):
        """
        Test ID: DAOS-707
        Test Description: Write Avocado Test to verify Array data after
                          pool/container disconnect/reconnect.
        :avocado: tags=all,object,full_regression,small,array_object
        """
        self.d_log.info("Writing the Array Dataset")
        record_index = 0
        transaction = []
        for dkey in range(self.no_of_dkeys):
            for akey in range(self.no_of_akeys):
                c_values = []
                value = ("{0}".format(str(akey)[0]) *
                         self.record_length[record_index])
                for item in range(self.array_size):
                    c_values.append(
                        (ctypes.create_string_buffer(value), len(value) + 1))
                c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
                c_akey = ctypes.create_string_buffer("akey {0}".format(akey))

                new_transaction = self.container.get_new_tx()
                self.ioreq.insert_array(c_dkey, c_akey, c_values,
                                        new_transaction)
                self.container.commit_tx(new_transaction)
                transaction.append(new_transaction)

                record_index = record_index + 1
                if record_index == len(self.record_length):
                    record_index = 0

        self.reconnect()

        self.d_log.info("Array Dataset Verification -- Started")
        record_index = 0
        transaction_index = 0
        for dkey in range(self.no_of_dkeys):
            for akey in range(self.no_of_akeys):
                indata = []
                value = ("{0}".format(str(akey)[0]) *
                         self.record_length[record_index])
                for item in range(self.array_size):
                    indata.append(value)
                c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
                c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
                c_rec_count = ctypes.c_uint(len(indata))
                c_rec_size = ctypes.c_size_t(len(indata[0]) + 1)

                outdata = self.ioreq.fetch_array(c_dkey, c_akey, c_rec_count,
                                                 c_rec_size)

                for item in enumerate(indata):
                    if indata[item[0]] != outdata[item[0]][:-1]:
                        self.d_log.error("ERROR:Data mismatch for "
                                         "dkey = {0}, "
                                         "akey = {1}".format(
                                             "dkey {0}".format(dkey),
                                             "akey {0}".format(akey)))
                        self.fail(
                            "ERROR:Data mismatch for dkey = {0}, akey={1}".
                            format("dkey {0}".format(dkey),
                                   "akey {0}".format(akey)))

                transaction_index = transaction_index + 1
                record_index = record_index + 1
                if record_index == len(self.record_length):
                    record_index = 0
Beispiel #3
0
class BasicSnapshot(TestWithServers):
    """DAOS-1370 Basic snapshot test.

    Test Class Description:
        Test that a snapshot taken of a container remains unchaged even after
        an object in the container has been updated 500 times.
        Create the container.
        Write an object to the container.
        Take a snapshot.
        Write 500 changes to the KV pair of the object.
        Check that the snapshot is still there.
        Confirm that the data in the snapshot is unchanged.
        Destroy the snapshot

    :avocado: recursive
    """

    def __init__(self, *args, **kwargs):
        """Initialize a BasicSnapshot object."""
        super(BasicSnapshot, self).__init__(*args, **kwargs)
        self.snapshot = None

    def test_basic_snapshot(self):
        """Test ID: DAOS-1370.

        Test Description:
            Create a pool, container in the pool, object in the container, add
            one key:value to the object.
            Commit the transaction. Perform a snapshot create on the container.
            Create 500 additional transactions with a small change to the object
            in each and commit each after the object update is done.
            Verify the snapshot is still available and the contents remain in
            their original state.

        :avocado: tags=snap,basicsnap
        """
        # Set up the pool and container.
        try:
            # initialize a pool object then create the underlying
            # daos storage, and connect
            self.prepare_pool()

            # create a container
            self.container = DaosContainer(self.context)
            self.container.create(self.pool.pool.handle)

            # now open it
            self.container.open()

        except DaosApiError as error:
            self.log.error(str(error))
            self.fail("Test failed before snapshot taken")

        try:
            # create an object and write some data into it
            obj_cls = self.params.get("obj_class", '/run/object_class/*')
            thedata = "Now is the winter of our discontent made glorious"
            datasize = len(thedata) + 1
            dkey = "dkey"
            akey = "akey"
            tx_handle = self.container.get_new_tx()
            obj = self.container.write_an_obj(thedata,
                                              datasize,
                                              dkey,
                                              akey,
                                              obj_cls=obj_cls,
                                              txn=tx_handle)
            self.container.commit_tx(tx_handle)
            obj.close()
            # Take a snapshot of the container
            self.snapshot = DaosSnapshot(self.context)
            self.snapshot.create(self.container.coh, tx_handle)
            self.log.info("Wrote an object and created a snapshot")

        except DaosApiError as error:
            self.fail("Test failed during the initial object write.\n{0}"
                      .format(error))

        # Make 500 changes to the data object. The write_an_obj function does a
        # commit when the update is complete
        try:
            self.log.info(
                "Committing 500 additional transactions to the same KV")
            more_transactions = 500
            while more_transactions:
                size = random.randint(1, 250) + 1
                new_data = get_random_string(size)
                new_obj = self.container.write_an_obj(
                    new_data, size, dkey, akey, obj_cls=obj_cls)
                new_obj.close()
                more_transactions -= 1
        except DaosApiError as error:
            self.fail(
                "Test failed during the write of 500 objects.\n{0}".format(
                    error))

        # List the snapshot
        try:
            reported_epoch = self.snapshot.list(self.container.coh)
        except DaosApiError as error:
            self.fail(
                "Test was unable to list the snapshot\n{0}".format(error))

        # Make sure the snapshot reflects the original epoch
        if self.snapshot.epoch != reported_epoch:
            self.fail(
                "The snapshot epoch returned from snapshot list is not the "
                "same as the original epoch snapshotted.")

        self.log.info(
            "After 500 additional commits the snapshot is still available")

        # Make sure the data in the snapshot is the original data.
        # Get a handle for the snapshot and read the object at dkey, akey.
        try:
            obj.open()
            snap_handle = self.snapshot.open(self.container.coh)
            thedata2 = self.container.read_an_obj(
                datasize, dkey, akey, obj, txn=snap_handle.value)
        except DaosApiError as error:
            self.fail(
                "Error when retrieving the snapshot data.\n{0}".format(error))

        # Compare the snapshot to the originally written data.
        if thedata2.value != thedata:
            self.fail(
                "The data in the snapshot is not the same as the original data")

        self.log.info(
            "The snapshot data matches the data originally written.")

        # Now destroy the snapshot
        try:
            self.snapshot.destroy(self.container.coh)
            self.log.info("Snapshot successfully destroyed")

        except DaosApiError as error:
            self.fail(str(error))
Beispiel #4
0
class PunchTest(TestWithServers):
    """
    Simple test to verify the 3 different punch calls.
    :avocado: recursive
    """
    def setUp(self):
        super(PunchTest, self).setUp()
        self.prepare_pool()

        try:
            # create a container
            self.container = DaosContainer(self.context)
            self.container.create(self.pool.pool.handle)

            # now open it
            self.container.open()
        except DaosApiError as excpn:
            print(excpn)
            print(traceback.format_exc())
            self.fail("Test failed during setup.\n")

    def test_dkey_punch(self):
        """
        The most basic test of the dkey punch function.

        :avocado: tags=all,object,pr,small,dkeypunch
        """

        try:
            # create an object and write some data into it
            thedata = "a string that I want to stuff into an object"
            dkey = "this is the dkey"
            akey = "this is the akey"
            tx_handle = self.container.get_new_tx()
            print("Created a new TX for punch dkey test")

            obj = self.container.write_an_obj(thedata, len(thedata)+1, dkey,
                                              akey, obj_cls=1, txn=tx_handle)
            print("Committing the TX for punch dkey test")
            self.container.commit_tx(tx_handle)
            print("Committed the TX for punch dkey test")

            # read the data back and make sure its correct
            thedata2 = self.container.read_an_obj(len(thedata)+1, dkey, akey,
                                                  obj, txn=tx_handle)
            if thedata != thedata2.value:
                print("data I wrote:" + thedata)
                print("data I read back" + thedata2.value)
                self.fail("Wrote data, read it back, didn't match\n")

            # now punch this data, should fail, can't punch committed data
            obj.punch_dkeys(tx_handle, [dkey])

            # expecting punch of commit data above to fail
            self.fail("Punch should have failed but it didn't.\n")

        # expecting an exception so do nothing
        except DaosApiError as dummy_e:
            pass

        try:
            self.container.close_tx(tx_handle)
            print("Closed TX for punch dkey test")

            # now punch this data
            obj.punch_dkeys(0, [dkey])

        # this one should work so error if exception occurs
        except DaosApiError as dummy_e:
            self.fail("Punch should have worked.\n")

        # there are a bunch of other cases to test here,
        #    --test punching the same updating and punching the same data in
        #    the same tx, should fail
        #    --test non updated data in an open tx, should work

    def test_akey_punch(self):
        """
        The most basic test of the akey punch function.

        :avocado: tags=all,object,pr,small,akeypunch
        """

        try:
            # create an object and write some data into it
            dkey = "this is the dkey"
            data1 = [("this is akey 1", "this is data value 1"),
                     ("this is akey 2", "this is data value 2"),
                     ("this is akey 3", "this is data value 3")]
            tx_handle = self.container.get_new_tx()
            print("Created a new TX for punch akey test")
            obj = self.container.write_multi_akeys(dkey, data1, obj_cls=1,
                                                   txn=tx_handle)
            print("Committing the TX for punch akey test")
            self.container.commit_tx(tx_handle)
            print("Committed the TX for punch dkey test")

            # read back the 1st epoch's data and check 1 value just to make sure
            # everything is on the up and up
            readbuf = [(data1[0][0], len(data1[0][1]) + 1),
                       (data1[1][0], len(data1[1][1]) + 1),
                       (data1[2][0], len(data1[2][1]) + 1)]
            retrieved_data = self.container.read_multi_akeys(dkey, readbuf, obj,
                                                             txn=tx_handle)
            if retrieved_data[data1[1][0]] != data1[1][1]:
                print("middle akey: {}".format(retrieved_data[data1[1][0]]))
                self.fail("data retrieval failure")

            # now punch one akey from this data
            obj.punch_akeys(tx_handle, dkey, [data1[1][0]])

            # expecting punch of commit data above to fail
            self.fail("Punch should have failed but it didn't.\n")

        # expecting an exception so do nothing
        except DaosApiError as excep:
            print(excep)

        try:
            self.container.close_tx(tx_handle)
            print("Closed TX for punch akey test")

            # now punch the object without a tx
            obj.punch_akeys(0, dkey, [data1[1][0]])

        # expecting it to work this time so error
        except DaosApiError as excep:
            self.fail("Punch should have worked: {}\n".format(excep))

    def test_obj_punch(self):
        """
        The most basic test of the object punch function.  Really similar
        to above except the whole object is deleted.

        :avocado: tags=all,object,pr,small,objpunch
        """

        try:

            # create an object and write some data into it
            thedata = "a string that I want to stuff into an object"
            dkey = "this is the dkey"
            akey = "this is the akey"
            tx_handle = self.container.get_new_tx()
            print("Created a new TX for punch obj test")
            obj = self.container.write_an_obj(thedata, len(thedata)+1, dkey,
                                              akey, obj_cls=1, txn=tx_handle)
            print("Committing the TX for punch obj test")
            self.container.commit_tx(tx_handle)
            print("Committed the TX for punch obj test")
            # read the data back and make sure its correct
            thedata2 = self.container.read_an_obj(len(thedata)+1, dkey, akey,
                                                  obj, txn=tx_handle)
            if thedata != thedata2.value:
                print("data I wrote:" + thedata)
                print("data I read back" + thedata2.value)
                self.fail("Wrote data, read it back, didn't match\n")

            # now punch the object, commited so not expecting it to work
            obj.punch(tx_handle)

            # expecting punch of commit data above to fail
            self.fail("Punch should have failed but it didn't.\n")

        # expecting an exception so do nothing
        except DaosApiError as excep:
            print(excep)

        try:
            self.container.close_tx(tx_handle)
            print("Closed TX for punch obj test")

            obj.punch(0)

        # expecting it to work without a tx
        except DaosApiError as excep:
            print(excep)
            self.fail("Punch should have worked.\n")
Beispiel #5
0
class Snapshot(TestWithServers):
    """
    Epic: DAOS-2249 Create system level tests that cover basic snapshot
          functionality.
    Testcase:
          DAOS-1370 Basic snapshot test
          DAOS-1386 Test container SnapShot information
          DAOS-1371 Test list snapshots
          DAOS-1395 Test snapshot destroy
          DAOS-1402 Test creating multiple snapshots

    Test Class Description:
          Start DAOS servers, set up the pool and container for the above
          snapshot Epic and Testcases, including snapshot basic, container
          information, list, creation and destroy.
    :avocado: recursive
    """
    def setUp(self):
        """
        set up method
        """
        super(Snapshot, self).setUp()
        # get parameters from yaml file, set default
        createmode = self.params.get("mode", '/run/poolparams/createmode/',
                                     511)
        createuid = os.geteuid()
        creategid = os.getegid()
        createsetid = self.params.get("setname", '/run/poolparams/createset/')
        createsize = self.params.get("size", '/run/poolparams/createsize/')
        self.log.info("==In setUp, self.context= %s", self.context)

        try:

            # initialize a python pool object then create the underlying
            # daos storage
            self.pool = DaosPool(self.context)
            self.pool.create(createmode, createuid, creategid, createsize,
                             createsetid, None)

            # need a connection to the pool with rw permission
            #    DAOS_PC_RO = int(1 << 0)
            #    DAOS_PC_RW = int(1 << 1)
            #    DAOS_PC_EX = int(1 << 2)
            self.pool.connect(1 << 1)

            # create a container
            self.container = DaosContainer(self.context)
            self.container.create(self.pool.handle)

        except DaosApiError as error:
            self.log.info("Error detected in DAOS pool container setup: %s",
                          str(error))
            self.log.info(traceback.format_exc())
            self.fail("##Test failed on setUp, before snapshot taken")

        # now open it
        self.container.open()

        # do a query and compare the UUID returned from create with
        # that returned by query
        self.container.query()

        if self.container.get_uuid_str() != c_uuid_to_str(
                self.container.info.ci_uuid):
            self.fail("##Container UUID did not match the one in info.")

    def display_snapshot(self, snapshot):
        """
        To display the snapshot information.
        Args:
            snapshot: snapshot handle to be displayed.
        Return:
            none.
        """
        self.log.info("==display_snapshot================")
        self.log.info("snapshot=                 %s", snapshot)
        self.log.info("snapshot.context=         %s", snapshot.context)
        self.log.info("snapshot.context.libdaos= %s", snapshot.context.libdaos)
        self.log.info("snapshot.context.libtest= %s", snapshot.context.libtest)
        self.log.info("snapshot.context.ftable= %s", snapshot.context.ftable)
        self.log.info("snapshot.context.ftable[list-attr]= %s",
                      snapshot.context.ftable["list-attr"])
        self.log.info("snapshot.context.ftable[test-event]=%s",
                      snapshot.context.ftable["test-event"])
        self.log.info("snapshot.name=  %s", snapshot.name)
        self.log.info("snapshot.epoch= %s", snapshot.epoch)
        self.log.info("==================================")

    def take_snapshot(self, container, epoch):
        """
        To take a snapshot on the container with current epoch.

        Args:
            container: container for the snapshot
            epoch: the container epoch for the snapshot
        Return:
            An object representing the snapshot
        """
        self.log.info("==Taking snapshot for:")
        self.log.info("    coh=   %s", container.coh)
        self.log.info("    epoch= %s", epoch)
        snapshot = DaosSnapshot(self.context)
        snapshot.create(container.coh, epoch)
        self.display_snapshot(snapshot)
        return snapshot

    def invalid_snapshot_test(self, coh, epoch):
        """
        Negative snapshot test with invalid container handle or epoch.

        Args:
            container: container for the snapshot
            epoch: the container epoch for the snapshot
        Return:
            0: Failed
            1: Passed (expected failure detected)
        """
        status = 0
        try:
            snapshot = DaosSnapshot(self.context)
            snapshot.create(coh, epoch)
        except Exception as error:
            self.log.info("==>Negative test, expected error: %s", str(error))
            status = 1
        return status

    def test_snapshot_negativecases(self):
        """
        Test ID: DAOS-1390 Verify snap_create bad parameter behavior.
                 DAOS-1322 Create a new container, verify snapshot state.
                           as expected for a brand new container.
                 DAOS-1392 Verify snap_destroy bad parameter behavior.
                 DAOS-1388 Verify snap_list bad parameter behavior.
        Test Description:
                (0)Take a snapshot of the newly created container.
                (1)Create an object, write random data into it, and take
                   a snapshot.
                (2)Verify the snapshot is working properly.
                (3)Test snapshot with an invalid container handle.
                (4)Test snapshot with a NULL container handle.
                (5)Test snapshot with an invalid epoch.
                (6)Verify snap_destroy with a bad parameter.
                (7)Verify snap_list bad parameter behavior.

        Use Cases: Combinations with minimun 1 client and 1 server.
        :avocado: tags=all,small,smoke,pr,snap,snapshot_negative,
        :avocado: tags=snapshotcreate_negative
        """

        #DAOS-1322 Create a new container, verify snapshot state as expected
        #          for a brand new container.
        try:
            self.log.info(
                "==(0)Take a snapshot of the newly created container.")
            snapshot = DaosSnapshot(self.context)
            snapshot.create(self.container.coh, 0)
            self.display_snapshot(snapshot)
        except Exception as error:
            self.fail("##(0)Error on a snapshot on a new container %s",
                      str(error))

        #(1)Create an object, write some data into it, and take a snapshot
        obj_cls = self.params.get("obj_class", '/run/object_class/*')
        akey = self.params.get("akey", '/run/snapshot/*', default="akey")
        dkey = self.params.get("dkey", '/run/snapshot/*', default="dkey")
        data_size = self.params.get("test_datasize",
                                    '/run/snapshot/*',
                                    default=150)
        rand_str = lambda n: ''.join(
            [random.choice(string.lowercase) for i in xrange(n)])
        thedata = "--->>>Happy Daos Snapshot-Create Negative Testing " + \
                  "<<<---" + rand_str(random.randint(1, data_size))
        try:
            tx_handle = self.container.get_new_tx()
            obj = self.container.write_an_obj(thedata,
                                              len(thedata) + 1,
                                              dkey,
                                              akey,
                                              obj_cls=obj_cls,
                                              txn=tx_handle)
            self.container.commit_tx(tx_handle)
        except DaosApiError as error:
            self.fail("##(1)Test failed during the initial object write: %s",
                      str(error))
        obj.close()
        ##Take a snapshot of the container
        snapshot = self.take_snapshot(self.container, tx_handle)
        self.log.info("==(1)Container epoch= %s", tx_handle)
        self.log.info("     snapshot.epoch= %s", snapshot.epoch)

        #(2)Verify the snapshot is working properly.
        try:
            obj.open()
            snap_handle = snapshot.open(self.container.coh, snapshot.epoch)
            thedata2 = self.container.read_an_obj(len(thedata) + 1,
                                                  dkey,
                                                  akey,
                                                  obj,
                                                  txn=snap_handle.value)
        except Exception as error:
            self.fail("##(2)Error when retrieving the snapshot data: %s",
                      str(error))
        self.log.info("==(2)snapshot_list[ind]=%s", snapshot)
        self.log.info("==snapshot.epoch=  %s", snapshot.epoch)
        self.log.info("==written thedata=%s", thedata)
        self.log.info("==thedata2.value= %s", thedata2.value)
        if thedata2.value != thedata:
            raise Exception("##(2)The data in the snapshot is not the "
                            "same as the original data")
        self.log.info("==Snapshot data matches the data originally "
                      "written.")

        #(3)Test snapshot with an invalid container handle
        self.log.info("==(3)Snapshot with an invalid container handle.")
        if self.invalid_snapshot_test(self.container, tx_handle):
            self.log.info(
                "==>Negative test 1, expecting failed on taking "
                "snapshot with an invalid container.coh: %s", self.container)
        else:
            self.fail(
                "##(3)Negative test 1 passing, expecting failed on"
                " taking snapshot with an invalid container.coh: %s",
                self.container)

        #(4)Test snapshot with a NULL container handle
        self.log.info("==(4)Snapshot with a NULL container handle.")
        if self.invalid_snapshot_test(None, tx_handle):
            self.log.info("==>Negative test 2, expecting failed on taking "
                          "snapshot on a NULL container.coh.")
        else:
            self.fail("##(4)Negative test 2 passing, expecting failed on "
                      "taking snapshot with a NULL container.coh.")

        #(5)Test snapshot with an invalid epoch
        self.log.info("==(5)Snapshot with a NULL epoch.")
        if self.invalid_snapshot_test(self.container.coh, None):
            self.log.info("==>Negative test 3, expecting failed on taking "
                          "snapshot with a NULL epoch.")
        else:
            self.fail("##(5)Negative test 3 passing, expecting failed on "
                      "taking snapshot with a NULL epoch.")

        #(6)DAOS-1392 destroy snapshot with an invalid handle
        self.log.info(
            "==(6)DAOS-1392 destroy snapshot with an invalid handle.")
        try:
            snapshot.destroy(None, tx_handle)
            self.fail("##(6)Negative test destroy snapshot with an "
                      "invalid coh handle, expected fail, shown Passing##")
        except Exception as error:
            self.log.info(
                "==>Negative test, destroy snapshot with an invalid handle.")
            self.log.info("   Expected Error: %s", str(error))
            expected_error = "RC: -1002"
            if expected_error not in str(error):
                self.fail("##(6.1)Expecting error RC: -1002  did not show.")

        #(7)DAOS-1388 Verify snap_list bad parameter behavior
        self.log.info(
            "==(7)DAOS-1388 Verify snap_list bad parameter behavior.")
        try:
            snapshot.list(None, 0)
            self.fail("##(7)Negative test snapshot list with an "
                      "invalid coh and epoch, expected fail, shown Passing##")
        except Exception as error:
            self.log.info(
                "==>Negative test, snapshot list with an invalid coh.")
            self.log.info("   Expected Error: %s", str(error))
            expected_error = "RC: -1002"
            if expected_error not in str(error):
                self.fail("##(7.1)Expecting error RC: -1002  did not show.")

    def display_snapshot_test_data(self, test_data, ss_index):
        """Display the snapshot test data.

        Args:
            test_data: list of snapshot testdata
                dictionary keys:
                    coh:             container handle
                    container_epoch: container epoch
                    snapshot:        snapshot handle
                    tst_obj:         test object
                    tst_data:        test data
            ss_index: snapshot-list index to be displayed.
        """
        if len(test_data) < ss_index - 1:
            self.log.info("##Under to display test_data info, "
                          "index out of range.")
        else:
            ind = ss_index - 1
            self.log.info("  =Snapshot number : %s", ss_index)
            self.log.info("  ==container_coh     =%s", test_data[ind]["coh"])
            self.log.info("  ==container_epoch   =%s",
                          test_data[ind]["container_epoch"])
            self.log.info("  ==snapshot          =%s",
                          test_data[ind]["snapshot"])
            self.log.info("  ==snapshot.epoch    =%s",
                          test_data[ind]["snapshot"].epoch)
            self.log.info("  ==data obj          =%s",
                          test_data[ind]["tst_obj"])
            self.log.info("  ==snapshot tst_data_size= %s",
                          len(test_data[ind]["tst_data"]) + 1)
            self.log.info("  ==original tst_data =%s",
                          test_data[ind]["tst_data"])
        return

    def test_snapshots(self):
        """
        Test ID: DAOS-1386 Test container SnapShot information
                 DAOS-1371 Test list snapshots
                 DAOS-1395 Test snapshot destroy
                 DAOS-1402 Test creating multiple snapshots
        Test Description:
                (1)Create an object, write random data into it, and take
                   a snapshot.
                (2)Make changes to the data object. The write_an_obj function
                   does a commit when the update is complete.
                (3)Verify the data in the snapshot is the original data.
                   Get a handle for the snapshot and read the object at dkey,
                   akey. Compare it to the originally written data.
                (4)List the snapshot and make sure it reflects the original
                   epoch.
                   ==>Repeat step(1) to step(4) for multiple snapshot tests.
                (5)Verify the snapshots data.
                (6)Destroy the snapshot individually.
                (7)Check if still able to Open the destroyed snapshot and
                   Verify the snapshot removed from the snapshot list.
                (8)Destroy the container snapshot.
        Use Cases: Require 1 client and 1 server to run snapshot test.
                   1 pool and 1 container is used, num_of_snapshot defined
                   in the snapshot.yaml will be performed and verified.
        :avocado: tags=all,small,smoke,snap,snapshots,full_regression
        """

        test_data = []
        ss_number = 0
        obj_cls = self.params.get("obj_class", '/run/object_class/*')
        akey = self.params.get("akey", '/run/snapshot/*', default="akey")
        dkey = self.params.get("dkey", '/run/snapshot/*', default="dkey")
        data_size = self.params.get("test_datasize",
                                    '/run/snapshot/*',
                                    default=150)
        snapshot_loop = self.params.get("num_of_snapshot",
                                        '/run/snapshot/*',
                                        default=3)
        rand_str = lambda n: ''.join(
            [random.choice(string.lowercase) for i in xrange(n)])
        #
        #Test loop for creat, modify and snapshot object in the DAOS container.
        #
        while ss_number < snapshot_loop:
            #(1)Create an object, write some data into it, and take a snapshot
            ss_number += 1
            thedata = "--->>>Happy Daos Snapshot Testing " + \
                str(ss_number) + \
                "<<<---" + rand_str(random.randint(1, data_size))
            datasize = len(thedata) + 1
            try:
                tx_handle = self.container.get_new_tx()
                obj = self.container.write_an_obj(thedata,
                                                  datasize,
                                                  dkey,
                                                  akey,
                                                  obj_cls=obj_cls,
                                                  txn=tx_handle)
                self.container.commit_tx(tx_handle)
                obj.close()
            except DaosApiError as error:
                self.fail("##(1)Test failed during the initial object "
                          "write: {}".format(str(error)))
            #Take a snapshot of the container
            snapshot = DaosSnapshot(self.context)
            snapshot.create(self.container.coh, tx_handle)
            self.log.info("==Wrote an object and created a snapshot")

            #Display snapshot
            self.log.info("=(1.%s)snapshot test loop: %s", ss_number,
                          ss_number)
            self.log.info("  ==self.container epoch=     %s", tx_handle)
            self.log.info("  ==snapshot.epoch= %s", snapshot.epoch)
            self.display_snapshot(snapshot)

            #Save snapshot test data
            test_data.append({
                "coh": self.container.coh,
                "container_epoch": tx_handle,
                "tst_obj": obj,
                "snapshot": snapshot,
                "tst_data": thedata
            })

            #(2)Make changes to the data object. The write_an_obj function does
            #   a commit when the update is complete
            num_transactions = more_transactions = 200
            self.log.info(
                "=(2.%s)Committing %d additional transactions to "
                "the same KV.", ss_number, more_transactions)
            while more_transactions:
                size = random.randint(1, 250) + 1
                new_data = rand_str(size)
                try:
                    new_obj = self.container.write_an_obj(new_data,
                                                          size,
                                                          dkey,
                                                          akey,
                                                          obj_cls=obj_cls)
                    new_obj.close()
                except Exception as error:
                    self.fail("##(2)Test failed during the write of "
                              "multi-objects: {}".format(str(error)))
                more_transactions -= 1

            #(3)Verify the data in the snapshot is the original data.
            #   Get a handle for the snapshot and read the object at dkey, akey
            #   Compare it to the originally written data.
            self.log.info("=(3.%s)snapshot test loop: %s", ss_number,
                          ss_number)
            try:
                obj.open()
                snap_handle = snapshot.open(self.container.coh, snapshot.epoch)
                thedata3 = self.container.read_an_obj(datasize,
                                                      dkey,
                                                      akey,
                                                      obj,
                                                      txn=snap_handle.value)
                obj.close()
            except Exception as error:
                self.fail("##(3.1)Error when retrieving the snapshot data: {}".
                          format(str(error)))
            self.display_snapshot_test_data(test_data, ss_number)
            self.log.info("  ==thedata3.value= %s", thedata3.value)
            if thedata3.value != thedata:
                raise Exception("##(3.2)The data in the snapshot is not the "
                                "same as the original data")
            self.log.info("  ==The snapshot data matches the data originally"
                          " written.")

            #(4)List the snapshot and make sure it reflects the original epoch
            try:
                ss_list = snapshot.list(self.container.coh, tx_handle)
                self.log.info("=(4.%s)snapshot.list(self.container.coh)= %s",
                              ss_number, ss_list)
                self.log.info("  ==snapshot.epoch=  %s", snapshot.epoch)
                self.log.info("  ==container epoch= %s", tx_handle)

            except Exception as error:
                self.fail(
                    "##(4)Test was unable to list the snapshot: {}".format(
                        str(error)))
            self.log.info(
                "  ==After %s additional commits the snapshot is "
                "still available", num_transactions)

        #(5)Verify the snapshots data
        for ind in range(len(test_data)):
            ss_number = ind + 1
            self.log.info("=(5.%s)Verify the snapshot number %s:", ss_number,
                          ss_number)
            self.display_snapshot_test_data(test_data, ss_number)
            coh = test_data[ind]["coh"]
            epoch = test_data[ind]["container_epoch"]
            current_ss = test_data[ind]["snapshot"]
            obj = test_data[ind]["tst_obj"]
            tst_data = test_data[ind]["tst_data"]
            datasize = len(tst_data) + 1
            try:
                obj.open()
                snap_handle5 = snapshot.open(coh, current_ss.epoch)
                thedata5 = self.container.read_an_obj(datasize,
                                                      dkey,
                                                      akey,
                                                      obj,
                                                      txn=snap_handle5.value)
                obj.close()
            except Exception as error:
                self.fail("##(5.1)Error when retrieving the snapshot data: {}".
                          format(str(error)))
            self.log.info("  ==snapshot tst_data =%s", thedata5.value)
            if thedata5.value != tst_data:
                raise Exception(
                    "##(5.2)Snapshot #%s, test data Mis-matches"
                    "the original data written.", ss_number)
            self.log.info(
                "  snapshot test number %s, test data matches"
                " the original data written.", ss_number)

            #(6)Destroy the individual snapshot
            self.log.info("=(6.%s)Destroy the snapshot epoch: %s", ss_number,
                          epoch)
            try:
                snapshot.destroy(coh, epoch)
                self.log.info("  ==snapshot epoch %s successfully destroyed",
                              epoch)
            except Exception as error:
                self.fail("##(6)Error on snapshot.destroy: {}".format(
                    str(error)))

        #(7)Check if still able to Open the destroyed snapshot and
        #   Verify the snapshot removed from the snapshot list
        try:
            obj.open()
            snap_handle7 = snapshot.open(coh, snapshot.epoch)
            thedata7 = self.container.read_an_obj(datasize,
                                                  dkey,
                                                  akey,
                                                  obj,
                                                  txn=snap_handle7.value)
            obj.close()
        except Exception as error:
            self.fail(
                "##(7)Error when retrieving the snapshot data: {}".format(
                    str(error)))
        self.log.info("=(7)=>thedata_after_snapshot.destroyed.value= %s",
                      thedata7.value)
        self.log.info("  ==>snapshot.epoch=     %s", snapshot.epoch)

        #Still able to open the snapshot and read data after destroyed.
        try:
            ss_list = snapshot.list(coh, epoch)
            self.log.info("  -->snapshot.list(coh, epoch)= %s", ss_list)
        except Exception as error:
            self.fail("##(7)Error when calling the snapshot list: {}".format(
                str(error)))

        #(8)Destroy the snapshot on the container
        try:
            snapshot.destroy(coh)
            self.log.info("=(8)Container snapshot destroyed successfully.")
        except Exception as error:
            self.fail("##(8)Error on snapshot.destroy. {}".format(str(error)))
        self.log.info("===DAOS container Multiple snapshots test passed.")
Beispiel #6
0
class ChecksumContainerValidation(TestWithServers):
    """
    Test Class Description: This test is enables
    checksum container properties and performs
    single object inserts and verifies
    contents. This is a basic sanity
    test for enabling checksum testing.
    This test case doesn't use TestPool/
    TestContainer for now. TestPool/TestContainer
    needs changes to support checksum.
    :avocado: recursive
    """

    # pylint: disable=too-many-instance-attributes
    def setUp(self):
        super(ChecksumContainerValidation, self).setUp()
        self.agent_sessions = None
        self.pool = None
        self.container = None
        self.obj = None
        self.ioreq = None
        self.no_of_dkeys = None
        self.no_of_akeys = None
        self.array_size = None
        self.record_length = None

        self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0]
        self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0]
        self.record_length = self.params.get("length", '/run/record/*')

        self.pool = TestPool(self.context)
        self.pool.get_params(self)
        self.pool.create()
        self.pool.connect(2)

        self.csum = self.params.get("enable_checksum", '/run/container/*')
        self.container = DaosContainer(self.context)
        input_param = self.container.cont_input_values
        input_param.enable_chksum = self.csum
        self.container.create(poh=self.pool.pool.handle, con_prop=input_param)
        self.container.open()

        self.obj = DaosObj(self.context, self.container)
        self.obj.create(objcls=1)
        self.obj.open()
        self.ioreq = IORequest(self.context,
                               self.container,
                               self.obj,
                               objtype=4)

    def test_single_object_with_checksum(self):
        """
        Test ID: DAOS-3927
        Test Description: Write Avocado Test to verify single data after
                          pool/container disconnect/reconnect.
        :avocado: tags=all,full_regression,pr,basic_checksum_object
        """
        self.d_log.info("Writing the Single Dataset")
        record_index = 0
        transaction = []
        for dkey in range(self.no_of_dkeys):
            for akey in range(self.no_of_akeys):
                indata = ("{0}".format(str(akey)[0]) *
                          self.record_length[record_index])
                c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
                c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
                c_value = ctypes.create_string_buffer(indata)
                c_size = ctypes.c_size_t(ctypes.sizeof(c_value))

                new_transaction = self.container.get_new_tx()
                self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
                                         new_transaction)
                self.container.commit_tx(new_transaction)
                transaction.append(new_transaction)
                record_index = record_index + 1
                if record_index == len(self.record_length):
                    record_index = 0

        self.d_log.info("Single Dataset Verification -- Started")
        record_index = 0
        for dkey in range(self.no_of_dkeys):
            for akey in range(self.no_of_akeys):
                indata = ("{0}".format(str(akey)[0]) *
                          self.record_length[record_index])
                c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
                c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
                val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1)
                if indata != (repr(val.value)[1:-1]):
                    self.d_log.error("ERROR:Data mismatch for "
                                     "dkey = {0}, "
                                     "akey = {1}".format(
                                         "dkey {0}".format(dkey),
                                         "akey {0}".format(akey)))
                    self.fail(
                        "ERROR: Data mismatch for dkey = {0}, akey={1}".format(
                            "dkey {0}".format(dkey), "akey {0}".format(akey)))

                record_index = record_index + 1
                if record_index == len(self.record_length):
                    record_index = 0
Beispiel #7
0
class CreateManyDkeys(Test):
    """
    Test Class Description:
        Tests that create large numbers of keys in objects/containers and then
        destroy the containers and verify the space has been reclaimed.

    :avocado: recursive
    """
    def setUp(self):
        super(CreateManyDkeys, self).setUp()
        self.pool = DaosPool(self.context)
        self.pool.create(self.params.get("mode", '/run/pool/createmode/*'),
                         os.geteuid(), os.getegid(),
                         self.params.get("size", '/run/pool/createsize/*'),
                         self.params.get("setname", '/run/pool/createset/*'),
                         None)
        self.pool.connect(1 << 1)

    def tearDown(self):
        try:
            if self.pool:
                self.pool.disconnect()
                self.pool.destroy(1)
        finally:
            super(CreateManyDkeys, self).tearDown()

    def write_a_bunch_of_values(self, how_many):
        """
        Write data to an object, each with a dkey and akey.  The how_many
        parameter determines how many key:value pairs are written.
        """

        self.container = DaosContainer(self.context)
        self.container.create(self.pool.handle)
        self.container.open()

        ioreq = IORequest(self.context, self.container, None)
        epoch = self.container.get_new_tx()
        c_epoch = ctypes.c_uint64(epoch)

        print("Started Writing the Dataset-----------\n")
        inc = 50000
        last_key = inc
        for key in range(how_many):
            c_dkey = ctypes.create_string_buffer("dkey {0}".format(key))
            c_akey = ctypes.create_string_buffer("akey {0}".format(key))
            c_value = ctypes.create_string_buffer(
                "some data that gets stored with the key {0}".format(key))
            c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
            ioreq.single_insert(c_dkey, c_akey, c_value, c_size, c_epoch)

            if key > last_key:
                print("written: {}".format(key))
                sys.stdout.flush()
                last_key = key + inc

        self.container.commit_tx(c_epoch)

        print("Started Verification of the Dataset-----------\n")
        last_key = inc
        for key in range(how_many):
            c_dkey = ctypes.create_string_buffer("dkey {0}".format(key))
            c_akey = ctypes.create_string_buffer("akey {0}".format(key))
            the_data = "some data that gets stored with the key {0}".format(
                key)
            val = ioreq.single_fetch(c_dkey, c_akey,
                                     len(the_data) + 1, c_epoch)

            if the_data != (repr(val.value)[1:-1]):
                self.fail("ERROR: Data mismatch for dkey = {0}, akey={1}, "
                          "Expected Value={2} and Received Value={3}\n".format(
                              "dkey {0}".format(key), "akey {0}".format(key),
                              the_data,
                              repr(val.value)[1:-1]))

            if key > last_key:
                print("veried: {}".format(key))
                sys.stdout.flush()
                last_key = key + inc

        print("starting destroy")
        self.container.close()
        self.container.destroy()
        print("destroy complete")

    @avocado.fail_on(DaosApiError)
    @skipForTicket("DAOS-1721")
    def test_many_dkeys(self):
        """
        Test ID: DAOS-1701
        Test Description: Test many of dkeys in same object.
        Use Cases: 1. large key counts
                   2. space reclaimation after destroy
        :avocado: tags=all,full,small,object,many_dkeys

        """

        no_of_dkeys = self.params.get("number_of_dkeys", '/run/dkeys/')

        # write a lot of individual data items, verify them, then destroy
        self.write_a_bunch_of_values(no_of_dkeys)

        # do it again, which should verify the first container
        # was truely destroyed because a second round won't fit otherwise
        self.write_a_bunch_of_values(no_of_dkeys)