Пример #1
0
class FullPoolContainerCreate(Test):
    """
    Class for test to create a container in a pool with no remaining free space.
    """

    def setUp(self):
        self.agent_sessions = None
        # get paths from the build_vars generated by build
        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                               "../../../../.build_vars.json")) as build_file:
            build_paths = json.load(build_file)
        self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../")
        self.server_group = self.params.get("name", '/server_config/',
                                            'daos_default_oops')

        self.context = DaosContext(build_paths['PREFIX'] + '/lib/')

        self.cont = None
        self.cont2 = None
        self.pool = DaosPool(self.context)
        self.d_log = DaosLog(self.context)
        self.hostlist = self.params.get("test_machines1", '/hosts/')
        self.hostfile = write_host_file.write_host_file(self.hostlist,
                                                        self.workdir)

        self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist)
        server_utils.run_server(self.hostfile, self.server_group, self.basepath)

    def tearDown(self):
        # shut 'er down
        """
        wrap pool destroy in a try; in case pool create didn't succeed, we
        still need the server to be shut down in any case
        """
        try:
            self.pool.destroy(1)
        finally:
            if self.agent_sessions:
                AgentUtils.stop_agent(self.hostlist, self.agent_sessions)
            server_utils.stop_server(hosts=self.hostlist)

    def test_no_space_cont_create(self):
        """
        :avocado: tags=pool,cont,fullpoolcontcreate,small,vm
        """
        # full storage rc
        err = "-1007"
        # probably should be -1007, revisit later
        err2 = "-1009"

        # create pool
        mode = self.params.get("mode", '/conttests/createmode/')
        self.d_log.debug("mode is {0}".format(mode))
        uid = os.geteuid()
        gid = os.getegid()
        # 16 mb pool, minimum size currently possible
        size = 16777216

        self.d_log.debug("creating pool")
        self.pool.create(mode, uid, gid, size, self.server_group, None)
        self.d_log.debug("created pool")

        # connect to the pool
        self.d_log.debug("connecting to pool")
        self.pool.connect(1 << 1)
        self.d_log.debug("connected to pool")

        # query the pool
        self.d_log.debug("querying pool info")
        dummy_pool_info = self.pool.pool_query()
        self.d_log.debug("queried pool info")

        # create a container
        try:
            self.d_log.debug("creating container")
            self.cont = DaosContainer(self.context)
            self.cont.create(self.pool.handle)
            self.d_log.debug("created container")
        except DaosApiError as excep:
            self.d_log.error("caught exception creating container: "
                             "{0}".format(excep))
            self.fail("caught exception creating container: {0}".format(excep))

        self.d_log.debug("opening container")
        self.cont.open()
        self.d_log.debug("opened container")

        # generate random dkey, akey each time
        # write 1mb until no space, then 1kb, etc. to fill pool quickly
        for obj_sz in [1048576, 1024, 1]:
            write_count = 0
            while True:
                self.d_log.debug("writing obj {0}, sz {1} to "
                                 "container".format(write_count, obj_sz))
                my_str = "a" * obj_sz
                my_str_sz = obj_sz
                dkey = (
                    ''.join(random.choice(string.lowercase) for i in range(5)))
                akey = (
                    ''.join(random.choice(string.lowercase) for i in range(5)))
                try:
                    dummy_oid, dummy_tx = self.cont.write_an_obj(my_str,
                                                                 my_str_sz,
                                                                 dkey, akey,
                                                                 obj_cls=1)
                    self.d_log.debug("wrote obj {0}, sz {1}".format(write_count,
                                                                    obj_sz))
                    write_count += 1
                except DaosApiError as excep:
                    if not (err in repr(excep) or err2 in repr(excep)):
                        self.d_log.error("caught exception while writing "
                                         "object: {0}".format(repr(excep)))
                        self.fail("caught exception while writing object: {0}"
                                  .format(repr(excep)))
                    else:
                        self.d_log.debug("pool is too full for {0} byte "
                                         "objects".format(obj_sz))
                        break

        self.d_log.debug("closing container")
        self.cont.close()
        self.d_log.debug("closed container")
        # create a 2nd container now that pool is full
        try:
            self.d_log.debug("creating 2nd container")
            self.cont2 = DaosContainer(self.context)
            self.cont2.create(self.pool.handle)
            self.d_log.debug("created 2nd container")

            self.d_log.debug("opening container 2")
            self.cont2.open()
            self.d_log.debug("opened container 2")

            self.d_log.debug("writing one more object, write expected to fail")
            self.cont2.write_an_obj(my_str, my_str_sz, dkey, akey, obj_cls=1)
            self.d_log.debug("wrote one more object--this should never print")
        except DaosApiError as excep:
            if not (err in repr(excep) or err2 in repr(excep)):
                self.d_log.error("caught unexpected exception while "
                                 "writing object: {0}".format(repr(excep)))
                self.fail("caught unexpected exception while writing "
                          "object: {0}".format(repr(excep)))
            else:
                self.d_log.debug("correctly caught -1007 while attempting "
                                 "to write object in full pool")
Пример #2
0
class Permission(Test):
    """
    Tests DAOS pool permissions while connect, whether
    modifying file with specific permissions work as expected.

    :avocado: tags=pool,permission
    """
    def setUp(self):
        # get paths from the build_vars generated by build
        with open('../../../.build_vars.json') as f:
            build_paths = json.load(f)
        self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../")
        self.server_group = self.params.get("server_group", '/server/',
                                            'daos_server')

        # setup the DAOS python API
        self.Context = DaosContext(build_paths['PREFIX'] + '/lib/')
        self.pool = None
        self.d_log = DaosLog(self.Context)

        # getting hostfile
        self.hostfile = None
        self.hostlist = self.params.get("test_machines", '/run/hosts/*')
        self.hostfile = WriteHostFile.WriteHostFile(self.hostlist,
                                                    self.workdir)
        print("Host file is: {}".format(self.hostfile))

        # starting server
        ServerUtils.runServer(self.hostfile, self.server_group, self.basepath)

    def tearDown(self):
        try:
            if self.pool is not None and self.pool.attached:
                self.pool.destroy(1)
        finally:
            # stop servers
            ServerUtils.stopServer(hosts=self.hostlist)

    def test_connectpermission(self):
        """
        Test pool connections with specific permissions.
        :avocado: tags=pool,permission,connectpermission
        """
        # parameters used in pool create
        createmode = self.params.get("mode", '/run/createtests/createmode/*/')
        createuid = os.geteuid()
        creategid = os.getegid()
        createsetid = self.params.get("setname", '/run/createtests/createset/')
        createsize = self.params.get("size", '/run/createtests/createsize/')

        # parameters used for pool connect
        permissions = self.params.get("perm", '/run/createtests/permissions/*')

        if (createmode == 73):
            expected_result = 'FAIL'
        if (createmode == 511 and permissions == 0):
            expected_result = 'PASS'
        elif (createmode in [146, 511] and permissions == 1):
            expected_result = 'PASS'
        elif (createmode in [292, 511] and permissions == 2):
            expected_result = 'PASS'
        else:
            expected_result = 'FAIL'

        try:
            # initialize a python pool object then create the underlying
            # daos storage
            self.pool = DaosPool(self.Context)
            self.d_log.debug("Pool initialisation successful")

            self.pool.create(createmode, createuid, creategid, createsize,
                             createsetid, None)
            self.d_log.debug("Pool Creation successful")

            self.pool.connect(1 << permissions)
            self.d_log.debug("Pool Connect successful")

            if expected_result in ['FAIL']:
                self.fail("Test was expected to fail but it passed.\n")

        except DaosApiError as e:
            print(e)
            if expected_result == 'PASS':
                self.fail("Test was expected to pass but it failed.\n")

    def test_filemodification(self):
        """
        Test whether file modification happens as expected under different
        permission levels.

        :avocado: tags=pool,permission,filemodification
        """
        # parameters used in pool create
        createmode = self.params.get("mode", '/run/createtests/createmode/*/')
        createuid = self.params.get("uid", '/run/createtests/createuid/')
        creategid = self.params.get("gid", '/run/createtests/creategid/')
        createsetid = self.params.get("setname", '/run/createtests/createset/')
        createsize = self.params.get("size", '/run/createtests/createsize/')

        if createmode == 73:
            expected_result = 'FAIL'
        elif createmode in [146, 511]:
            permissions = 1
            expected_result = 'PASS'
        elif createmode == 292:
            permissions = 2
            expected_result = 'PASS'

        try:
            # initialize a python pool object then create the underlying
            # daos storage
            self.pool = DaosPool(self.Context)
            self.d_log.debug("Pool initialisation successful")
            self.pool.create(createmode, createuid, creategid, createsize,
                             createsetid, None)
            self.d_log.debug("Pool Creation successful")

            self.pool.connect(1 << permissions)
            self.d_log.debug("Pool Connect successful")

            self.container = DaosContainer(self.Context)
            self.d_log.debug("Contianer initialisation successful")

            self.container.create(self.pool.handle)
            self.d_log.debug("Container create successful")

            # now open it
            self.container.open()
            self.d_log.debug("Container open successful")

            thedata = "a string that I want to stuff into an object"
            size = 45
            dkey = "this is the dkey"
            akey = "this is the akey"

            self.container.write_an_obj(thedata, size, dkey, akey)
            self.d_log.debug("Container write successful")
            if expected_result in ['FAIL']:
                self.fail("Test was expected to fail but it passed.\n")

        except DaosApiError as e:
            print(e)
            if expected_result == 'PASS':
                self.fail("Test was expected to pass but it failed.\n")
Пример #3
0
class FullPoolContainerCreate(Test):
    """
    Class for test to create a container in a pool with no remaining free space.
    """

    def setUp(self):
        # get paths from the build_vars generated by build
        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                "../../../../.build_vars.json")) as f:
            build_paths = json.load(f)
        self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../")
        self.server_group = self.params.get("server_group", '/server/',
                                            'daos_default_oops')

        self.context = DaosContext(build_paths['PREFIX'] + '/lib/')

        self.pool = DaosPool(self.context)
        self.d_log = DaosLog(self.context)
        self.hostlist = self.params.get("test_machines1", '/hosts/')
        self.hostfile = WriteHostFile.WriteHostFile(self.hostlist, self.workdir)
        ServerUtils.runServer(self.hostfile, self.server_group, self.basepath)

    def tearDown(self):
        # shut 'er down
        """
        wrap pool destroy in a try; in case pool create didn't succeed, we
        still need the server to be shut down in any case
        """
        try:
            self.pool.destroy(1)
        finally:
            ServerUtils.stopServer(hosts=self.hostlist)

    def test_no_space_cont_create(self):
        """
        :avocado: tags=pool,cont,fullpoolcontcreate,small,vm
        """
        setid = self.params.get("setname",
                                '/run/testparams/setnames/validsetname/')

        # full storage rc
        err = "-1007"
        # probably should be -1007, revisit later
        err2 = "-1009"

        # create pool
        mode = self.params.get("mode", '/conttests/createmode/')
        self.d_log.debug("mode is {0}".format(mode))
        uid = os.geteuid()
        gid = os.getegid()
        # 16 mb pool, minimum size currently possible
        size = 16777216

        self.d_log.debug("creating pool")
        self.pool.create(mode, uid, gid, size, self.server_group, None)
        self.d_log.debug("created pool")

        # connect to the pool
        self.d_log.debug("connecting to pool")
        self.pool.connect(1 << 1)
        self.d_log.debug("connected to pool")

        # query the pool
        self.d_log.debug("querying pool info")
        pool_info = self.pool.pool_query()
        self.d_log.debug("queried pool info")

        # create a container
        try:
            self.d_log.debug("creating container")
            self.cont = DaosContainer(self.context)
            self.cont.create(self.pool.handle)
            self.d_log.debug("created container")
        except DaosApiError as e:
            self.d_log.error("caught exception creating container: "
                             "{0}".format(e))
            self.fail("caught exception creating container: {0}".format(e))

        self.d_log.debug("opening container")
        self.cont.open()
        self.d_log.debug("opened container")

        # generate random dkey, akey each time
        # write 1mb until no space, then 1kb, etc. to fill pool quickly
        for x in [1048576, 1024, 1]:
            write_count = 0
            while(True):
                self.d_log.debug("writing obj {0}, sz {1} to "
                                 "container".format(write_count, x))
                my_str = "a" * x
                my_str_sz = x
                dkey = ''.join(random.choice(string.lowercase) for i in range(5))
                akey = ''.join(random.choice(string.lowercase) for i in range(5))
                try:
                    oid, tx = self.cont.write_an_obj(my_str, my_str_sz, dkey,
                                                        akey, obj_cls=1)
                    self.d_log.debug("wrote obj {0}, sz {1}".format(write_count,
                                                                    x))
                    write_count += 1
                except DaosApiError as e:
                    if not (err in repr(e) or err2 in repr(e)):
                        self.d_log.error("caught exception while writing "
                                         "object: {0}".format(repr(e)))
                        self.fail("caught exception while writing object: {0}"
                                  .format(repr(e)))
                    else:
                        self.d_log.debug("pool is too full for {0} byte "
                                         "objects".format(x))
                        break

        self.d_log.debug("closing container")
        self.cont.close()
        self.d_log.debug("closed container")
        # create a 2nd container now that pool is full
        try:
            self.d_log.debug("creating 2nd container")
            self.cont2 = DaosContainer(self.context)
            self.cont2.create(self.pool.handle)
            self.d_log.debug("created 2nd container")

            self.d_log.debug("opening container 2")
            self.cont2.open()
            self.d_log.debug("opened container 2")

            self.d_log.debug("writing one more object, write expected to fail")
            self.cont2.write_an_obj(my_str, my_str_sz, dkey, akey, obj_cls=1)
            self.d_log.debug("wrote one more object--this should never print")
        except DaosApiError as e:
            if not (err in repr(e) or err2 in repr(e)):
                    self.d_log.error("caught unexpected exception while "
                                     "writing object: {0}".format(repr(e)))
                    self.fail("caught unexpected exception while writing "
                              "object: {0}".format(repr(e)))
            else:
                self.d_log.debug("correctly caught -1007 while attempting "
                                 "to write object in full pool")
Пример #4
0
class ObjectMetadata(avocado.Test):
    """
    Test Class Description:
        Test the general Metadata operations and boundary conditions.
    """

    def setUp(self):
        self.agent_sessions = None
        self.pool = None
        self.hostlist = None
        self.hostfile_clients = None
        self.hostfile = None
        self.out_queue = None
        self.pool_connect = True

        with open('../../../.build_vars.json') as json_f:
            build_paths = json.load(json_f)

        self.basepath = os.path.normpath(build_paths['PREFIX']  + "/../")
        self.server_group = self.params.get("name",
                                            '/server_config/',
                                            'daos_server')
        self.context = DaosContext(build_paths['PREFIX'] + '/lib/')
        self.d_log = DaosLog(self.context)
        self.hostlist = self.params.get("servers", '/run/hosts/*')
        self.hostfile = write_host_file.write_host_file(self.hostlist,
                                                        self.workdir)
        self.hostlist_clients = self.params.get("clients", '/run/hosts/*')
        self.hostfile_clients = (
            write_host_file.write_host_file(hostlist_clients, self.workdir))
        self.agent_sessions = AgentUtils.run_agent(self.basepath, self.hostlist,
                                                   self.hostlist_clients)
        server_utils.run_server(self.hostfile, self.server_group, self.basepath)

        self.pool = DaosPool(self.context)
        self.pool.create(self.params.get("mode", '/run/pool/createmode/*'),
                         os.geteuid(),
                         os.getegid(),
                         self.params.get("size", '/run/pool/createsize/*'),
                         self.params.get("setname", '/run/pool/createset/*'),
                         nvme_size=self.params.get("size",
                                                   '/run/pool/nvmesize/*'))

    def tearDown(self):
        try:
            if self.pool_connect:
                self.pool.disconnect()
            if self.pool:
                self.pool.destroy(1)
        finally:
            if self.agent_sessions:
                AgentUtils.stop_agent(self.hostlist_clients,
                                      self.agent_sessions)
            server_utils.stop_server(hosts=self.hostlist)

    @avocado.skip("Skipping until DAOS-1936/DAOS-1946 is fixed.")
    def test_metadata_fillup(self):
        """
        Test ID: DAOS-1512
        Test Description: Test to verify no IO happens after metadata is full.
        :avocado: tags=metadata,metadata_fill,nvme,small
        """
        self.pool.connect(2)
        container = DaosContainer(self.context)
        self.d_log.debug("Fillup Metadata....")
        for _cont in range(NO_OF_MAX_CONTAINER):
            container.create(self.pool.handle)
        self.d_log.debug("Metadata Overload...")
        #This should fail with no Metadata space Error.
        try:
            for _cont in range(250):
                container.create(self.pool.handle)
        except DaosApiError as exe:
            print (exe, traceback.format_exc())
            return

        self.fail("Test was expected to fail but it passed.\n")

    @avocado.skip("Skipping until DAOS-1965 is fixed.")
    @avocado.fail_on(DaosApiError)
    def test_metadata_addremove(self):
        """
        Test ID: DAOS-1512
        Test Description: Verify metadata release the space
                          after container delete.
        :avocado: tags=metadata,metadata_free_space,nvme,small
        """
        self.pool.connect(2)
        for k in range(10):
            container_array = []
            self.d_log.debug("Container Create Iteration {}".format(k))
            for cont in range(NO_OF_MAX_CONTAINER):
                container = DaosContainer(self.context)
                container.create(self.pool.handle)
                container_array.append(container)
            self.d_log.debug("Container Remove Iteration {} ".format(k))
            for cont in container_array:
                cont.destroy()

    def thread_control(self, threads, operation):
        """
        Start threads and wait till all threads execution is finished.
        It check queue for "FAIL" message and fail the avocado test.
        """
        self.d_log.debug("IOR {0} Threads Started -----".format(operation))
        for thrd in threads:
            thrd.start()
        for thrd in threads:
            thrd.join()

        while not self.out_queue.empty():
            if self.out_queue.get() == "FAIL":
                return "FAIL"
        self.d_log.debug("IOR {0} Threads Finished -----".format(operation))
        return "PASS"

    @avocado.fail_on(DaosApiError)
    def test_metadata_server_restart(self):
        """
        Test ID: DAOS-1512
        Test Description: This test will verify 2000 IOR small size container
                          after server restart. Test will write IOR in 5
                          different threads for faster execution time. Each
                          thread will create 400 (8bytes) containers to the
                          same pool. Restart the servers, read IOR container
                          file written previously and validate data integrity
                          by using IOR option "-R -G 1".
        :avocado: tags=metadata,metadata_ior,nvme,small
        """
        self.pool_connect = False
        files_per_thread = 400
        total_ior_threads = 5
        threads = []
        ior_args = {}

        createsvc = self.params.get("svcn", '/run/pool/createsvc/')
        svc_list = ""
        for i in range(createsvc):
            svc_list += str(int(self.pool.svc.rl_ranks[i])) + ":"
        svc_list = svc_list[:-1]

        ior_args['client_hostfile'] = self.hostfile_clients
        ior_args['pool_uuid'] = self.pool.get_uuid_str()
        ior_args['svc_list'] = svc_list
        ior_args['basepath'] = self.basepath
        ior_args['server_group'] = self.server_group
        ior_args['tmp_dir'] = self.workdir
        ior_args['iorwriteflags'] = self.params.get("F",
                                                    '/run/ior/iorwriteflags/')
        ior_args['iorreadflags'] = self.params.get("F",
                                                   '/run/ior/iorreadflags/')
        ior_args['iteration'] = self.params.get("iter", '/run/ior/iteration/')
        ior_args['stripe_size'] = self.params.get("s", '/run/ior/stripesize/*')
        ior_args['stripe_count'] = self.params.get("c", '/run/ior/stripecount/')
        ior_args['async_io'] = self.params.get("a", '/run/ior/asyncio/')
        ior_args['object_class'] = self.params.get("o", '/run/ior/objectclass/')
        ior_args['slots'] = self.params.get("slots", '/run/ior/clientslots/*')

        ior_args['files_per_thread'] = files_per_thread
        self.out_queue = Queue.Queue()

        #IOR write threads
        for i in range(total_ior_threads):
            threads.append(threading.Thread(target=ior_runner_thread,
                                            args=(self.out_queue,
                                                  "Thread-{}".format(i),
                                                  "write"),
                                            kwargs=ior_args))
        if self.thread_control(threads, "write") == "FAIL":
            self.d_log.error(" IOR write Thread FAIL")
            self.fail(" IOR write Thread FAIL")

        #Server Restart
        if self.agent_sessions:
            AgentUtils.stop_agent(self.hostlist_clients, self.agent_sessions)
        server_utils.stop_server(hosts=self.hostlist)
        self.agent_sessions = AgentUtils.run_agent(self.basepath,
                                                   self.hostlist_clients,
                                                   self.hostlist)
        server_utils.run_server(self.hostfile, self.server_group, self.basepath)

        #Read IOR with verification with same number of threads
        threads = []
        for i in range(total_ior_threads):
            threads.append(threading.Thread(target=ior_runner_thread,
                                            args=(self.out_queue,
                                                  "Thread-{}".format(i),
                                                  "read"),
                                            kwargs=ior_args))
        if self.thread_control(threads, "read") == "FAIL":
            self.d_log.error(" IOR write Thread FAIL")
            self.fail(" IOR read Thread FAIL")
Пример #5
0
class ObjectMetadata(avocado.Test):
    """
    Test Class Description:
        Test the general Metadata operations and boundary conditions.
    """
    def setUp(self):
        self.pool = None
        self.hostlist = None
        self.hostfile_clients = None
        self.hostfile = None
        self.out_queue = None
        self.pool_connect = True

        with open('../../../.build_vars.json') as json_f:
            build_paths = json.load(json_f)

        self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../")
        self.server_group = self.params.get("server_group", '/server/',
                                            'daos_server')
        self.context = DaosContext(build_paths['PREFIX'] + '/lib/')
        self.d_log = DaosLog(self.context)
        self.hostlist = self.params.get("servers", '/run/hosts/*')
        self.hostfile = WriteHostFile.WriteHostFile(self.hostlist,
                                                    self.workdir)
        hostlist_clients = self.params.get("clients", '/run/hosts/*')
        self.hostfile_clients = WriteHostFile.WriteHostFile(
            hostlist_clients, self.workdir)
        ServerUtils.runServer(self.hostfile, self.server_group, self.basepath)

        self.pool = DaosPool(self.context)
        self.pool.create(self.params.get("mode", '/run/pool/createmode/*'),
                         os.geteuid(),
                         os.getegid(),
                         self.params.get("size", '/run/pool/createsize/*'),
                         self.params.get("setname", '/run/pool/createset/*'),
                         nvme_size=self.params.get("size",
                                                   '/run/pool/nvmesize/*'))

    def tearDown(self):
        try:
            if self.pool_connect:
                self.pool.disconnect()
            if self.pool:
                self.pool.destroy(1)
        finally:
            ServerUtils.stopServer(hosts=self.hostlist)

    @avocado.skip("Skipping until DAOS-1936/DAOS-1946 is fixed.")
    def test_metadata_fillup(self):
        """
        Test ID: DAOS-1512
        Test Description: Test to verify no IO happens after metadata is full.
        :avocado: tags=metadata,metadata_fill,nvme,small
        """
        self.pool.connect(2)
        container = DaosContainer(self.context)
        self.d_log.debug("Fillup Metadata....")
        for _cont in range(NO_OF_MAX_CONTAINER):
            container.create(self.pool.handle)
        self.d_log.debug("Metadata Overload...")
        #This should fail with no Metadata space Error.
        try:
            for _cont in range(250):
                container.create(self.pool.handle)
        except DaosApiError as exe:
            print(exe, traceback.format_exc())
            return

        self.fail("Test was expected to fail but it passed.\n")

    @avocado.skip("Skipping until DAOS-1965 is fixed.")
    @avocado.fail_on(DaosApiError)
    def test_metadata_addremove(self):
        """
        Test ID: DAOS-1512
        Test Description: Verify metadata release the space
                          after container delete.
        :avocado: tags=metadata,metadata_free_space,nvme,small
        """
        self.pool.connect(2)
        for k in range(10):
            container_array = []
            self.d_log.debug("Container Create Iteration {}".format(k))
            for cont in range(NO_OF_MAX_CONTAINER):
                container = DaosContainer(self.context)
                container.create(self.pool.handle)
                container_array.append(container)
            self.d_log.debug("Container Remove Iteration {} ".format(k))
            for cont in container_array:
                cont.destroy()

    def thread_control(self, threads, operation):
        """
        Start threads and wait till all threads execution is finished.
        It check queue for "FAIL" message and fail the avocado test.
        """
        self.d_log.debug("IOR {0} Threads Started -----".format(operation))
        for thrd in threads:
            thrd.start()
        for thrd in threads:
            thrd.join()

        while not self.out_queue.empty():
            if self.out_queue.get() is "FAIL":
                return "FAIL"
        self.d_log.debug("IOR {0} Threads Finished -----".format(operation))
        return "PASS"

    @avocado.fail_on(DaosApiError)
    def test_metadata_server_restart(self):
        """
        Test ID: DAOS-1512
        Test Description: This test will verify 2000 IOR small size container after server restart.
                          Test will write IOR in 5 different threads for faster execution time.
                          Each thread will create 400 (8bytes) containers to the same pool.
                          Restart the servers, read IOR container file written previously and
                          validate data integrity by using IOR option "-R -G 1".
        :avocado: tags=metadata,metadata_ior,nvme,small
        """
        self.pool_connect = False
        files_per_thread = 400
        total_ior_threads = 5
        threads = []
        ior_args = {}

        createsvc = self.params.get("svcn", '/run/pool/createsvc/')
        svc_list = ""
        for i in range(createsvc):
            svc_list += str(int(self.pool.svc.rl_ranks[i])) + ":"
        svc_list = svc_list[:-1]

        ior_args['client_hostfile'] = self.hostfile_clients
        ior_args['pool_uuid'] = self.pool.get_uuid_str()
        ior_args['svc_list'] = svc_list
        ior_args['basepath'] = self.basepath
        ior_args['server_group'] = self.server_group
        ior_args['tmp_dir'] = self.workdir
        ior_args['iorwriteflags'] = self.params.get("F",
                                                    '/run/ior/iorwriteflags/')
        ior_args['iorreadflags'] = self.params.get("F",
                                                   '/run/ior/iorreadflags/')
        ior_args['iteration'] = self.params.get("iter", '/run/ior/iteration/')
        ior_args['stripe_size'] = self.params.get("s", '/run/ior/stripesize/*')
        ior_args['stripe_count'] = self.params.get("c",
                                                   '/run/ior/stripecount/')
        ior_args['async_io'] = self.params.get("a", '/run/ior/asyncio/')
        ior_args['object_class'] = self.params.get("o",
                                                   '/run/ior/objectclass/')
        ior_args['slots'] = self.params.get("slots", '/run/ior/clientslots/*')

        ior_args['files_per_thread'] = files_per_thread
        self.out_queue = Queue.Queue()

        #IOR write threads
        for i in range(total_ior_threads):
            threads.append(
                threading.Thread(target=ior_runner_thread,
                                 args=(self.out_queue, "Thread-{}".format(i),
                                       "write"),
                                 kwargs=ior_args))
        if self.thread_control(threads, "write") == "FAIL":
            self.d_log.error(" IOR write Thread FAIL")
            self.fail(" IOR write Thread FAIL")

        #Server Restart
        ServerUtils.stopServer(hosts=self.hostlist)
        ServerUtils.runServer(self.hostfile, self.server_group, self.basepath)

        #Read IOR with verification with same number of threads
        threads = []
        for i in range(total_ior_threads):
            threads.append(
                threading.Thread(target=ior_runner_thread,
                                 args=(self.out_queue, "Thread-{}".format(i),
                                       "read"),
                                 kwargs=ior_args))
        if self.thread_control(threads, "read") == "FAIL":
            self.d_log.error(" IOR write Thread FAIL")
            self.fail(" IOR read Thread FAIL")