def test_createasync(self): """ Test container create for asynchronous mode. :avocado: tags=container,containerasync,createasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode",'/run/createtests/createmode/*/') createsetid = self.params.get("setname",'/run/createtests/createset/') createsize = self.params.get("size",'/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.POOL = DaosPool(self.Context) self.POOL.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.POOL.handle self.POOL.connect(1 << 1) # Container initialization and creation self.Container1 = DaosContainer(self.Context) self.Container2 = DaosContainer(self.Context) GLOB_SIGNAL = threading.Event() self.Container1.create(poh, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test") print ("RC after successful Container create: " , GLOB_RC) # Try to recreate container after destroying pool, # this should fail. Checking rc after failure. self.POOL.destroy(1) GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.Container2.create(poh, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != -1005: self.fail("RC not as expected in async test") print ("RC after Container create failed:", GLOB_RC) # cleanup the Pool and Container self.POOL = None except ValueError as e: print e print traceback.format_exc()
def check_handle(buf_len, iov_len, buf, uuidstr, rank): """ This gets run in a child process and verifyes the global handle can be turned into a local handle in another process. """ try: # get paths from the build_vars generated by build with open('../../../.build_vars.json') as build_file: build_paths = json.load(build_file) # setup the DAOS python API in this process context = DaosContext(build_paths['PREFIX'] + '/lib/') pool = DaosPool(context) pool.set_uuid_str(uuidstr) pool.set_svc(rank) pool.group = "daos_server" # note that the handle is stored inside the pool as well dummy_local_handle = pool.global2local(context, iov_len, buf_len, buf) # perform some operations that will use the new handle pool.pool_query() container = DaosContainer(context) container.create(pool.handle) except DaosApiError as excep: print(excep) print(traceback.format_exc()) raise return
def test_metadata_fillup(self): """JIRA ID: DAOS-1512. Test Description: Test to verify no IO happens after metadata is full. Use Cases: ? :avocado: tags=metadata,metadata_fill,nvme,small """ self.pool.connect(2) container = DaosContainer(self.context) self.d_log.debug("Fillup Metadata....") for _cont in range(NO_OF_MAX_CONTAINER): container.create(self.pool.handle) # This should fail with no Metadata space Error. self.d_log.debug("Metadata Overload...") try: for _cont in range(250): container.create(self.pool.handle) self.fail("Test expected to fail with a no metadata space error") except DaosApiError as exe: print(exe, traceback.format_exc()) return self.fail("Test was expected to fail but it passed.\n")
def setUp(self): try: super(PunchTest, self).setUp() # parameters used in pool create createmode = self.params.get("mode", '/run/pool/createmode/') createsetid = self.params.get("setname", '/run/pool/createset/') createsize = self.params.get("size", '/run/pool/createsize/') createuid = os.geteuid() creategid = os.getegid() # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) self.pool.connect(1 << 1) # create a container self.container = DaosContainer(self.context) self.container.create(self.pool.handle) # now open it self.container.open() except DaosApiError as excpn: print(excpn) print(traceback.format_exc()) self.fail("Test failed during setup.\n")
def setUp(self): with open('../../../.build_vars.json') as json_f: build_paths = json.load(json_f) basepath = os.path.normpath(build_paths['PREFIX'] + "/../") tmp = build_paths['PREFIX'] + '/tmp' server_group = self.params.get("server_group", '/server/', 'daos_server') self.context = DaosContext(build_paths['PREFIX'] + '/lib/') self.hostlist = self.params.get("test_machines", '/run/hosts/*') self.hostfile = WriteHostFile.WriteHostFile(self.hostlist, tmp) ServerUtils.runServer(self.hostfile, server_group, basepath) self.pool = DaosPool(self.context) self.pool.create(self.params.get("mode", '/run/pool/createmode/*'), os.geteuid(), os.getegid(), self.params.get("size", '/run/pool/createsize/*'), self.params.get("setname", '/run/pool/createset/*'), None) self.pool.connect(1 << 1) poh = self.pool.handle self.container = DaosContainer(self.context) self.container.create(poh) self.container.open()
def test_filemodification(self): """ Test whether file modification happens as expected under different permission levels. :avocado: tags=pool,permission,filemodification """ # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createuid = self.params.get("uid", '/run/createtests/createuid/') creategid = self.params.get("gid", '/run/createtests/creategid/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') if createmode == 73: expected_result = 'FAIL' elif createmode in [146, 511]: permissions = 1 expected_result = 'PASS' elif createmode == 292: permissions = 2 expected_result = 'PASS' try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.Context) self.d_log.debug("Pool initialisation successful") self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) self.d_log.debug("Pool Creation successful") self.pool.connect(1 << permissions) self.d_log.debug("Pool Connect successful") self.container = DaosContainer(self.Context) self.d_log.debug("Contianer initialisation successful") self.container.create(self.pool.handle) self.d_log.debug("Container create successful") # now open it self.container.open() self.d_log.debug("Container open successful") thedata = "a string that I want to stuff into an object" size = 45 dkey = "this is the dkey" akey = "this is the akey" self.container.write_an_obj(thedata, size, dkey, akey) self.d_log.debug("Container write successful") if expected_result in ['FAIL']: self.fail("Test was expected to fail but it passed.\n") except DaosApiError as e: print(e) if expected_result == 'PASS': self.fail("Test was expected to pass but it failed.\n")
def write_a_bunch_of_values(self, how_many): """ Write data to an object, each with a dkey and akey. The how_many parameter determines how many key:value pairs are written. """ self.container = DaosContainer(self.context) self.container.create(self.pool.handle) self.container.open() ioreq = IORequest(self.context, self.container, None) epoch = self.container.get_new_tx() c_epoch = ctypes.c_uint64(epoch) print("Started Writing the Dataset-----------\n") inc = 50000 last_key = inc for key in range(how_many): c_dkey = ctypes.create_string_buffer("dkey {0}".format(key)) c_akey = ctypes.create_string_buffer("akey {0}".format(key)) c_value = ctypes.create_string_buffer( "some data that gets stored with the key {0}".format(key)) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) ioreq.single_insert(c_dkey, c_akey, c_value, c_size, c_epoch) if key > last_key: print("written: {}".format(key)) sys.stdout.flush() last_key = key + inc self.container.commit_tx(c_epoch) print("Started Verification of the Dataset-----------\n") last_key = inc for key in range(how_many): c_dkey = ctypes.create_string_buffer("dkey {0}".format(key)) c_akey = ctypes.create_string_buffer("akey {0}".format(key)) the_data = "some data that gets stored with the key {0}".format( key) val = ioreq.single_fetch(c_dkey, c_akey, len(the_data) + 1, c_epoch) if the_data != (repr(val.value)[1:-1]): self.fail("ERROR: Data mismatch for dkey = {0}, akey={1}, " "Expected Value={2} and Received Value={3}\n".format( "dkey {0}".format(key), "akey {0}".format(key), the_data, repr(val.value)[1:-1])) if key > last_key: print("veried: {}".format(key)) sys.stdout.flush() last_key = key + inc print("starting destroy") self.container.close() self.container.destroy() print("destroy complete")
def check_handle(pool_glob_handle, uuidstr, cont_glob_handle, rank): """ This gets run in a child process and verifyes the global handles can be turned into local handles in another process. """ try: # get paths from the build_vars generated by build with open('../../../.build_vars.json') as build_file: build_paths = json.load(build_file) # setup the DAOS python API in this process context = DaosContext(build_paths['PREFIX'] + '/lib/') # setup the pool and connect using global handle pool = DaosPool(context) pool.uuid = uuidstr pool.set_svc(rank) pool.group = "daos_server" buf = ctypes.cast(pool_glob_handle.iov_buf, ctypes.POINTER(ctypes.c_byte * pool_glob_handle.iov_buf_len)) buf2 = bytearray() buf2.extend(buf.contents) pool_handle = pool.global2local(context, pool_glob_handle.iov_len, pool_glob_handle.iov_buf_len, buf2) # perform an operation that will use the new handle, if it # doesn't throw an exception, then all is well. pool.pool_query() # setup the container and then connect using the global handle container = DaosContainer(context) container.poh = pool_handle buf = ctypes.cast(cont_glob_handle.iov_buf, ctypes.POINTER(ctypes.c_byte * cont_glob_handle.iov_buf_len)) buf2 = bytearray() buf2.extend(buf.contents) dummy_cont_handle = container.global2local(context, cont_glob_handle.iov_len, cont_glob_handle.iov_buf_len, buf2) # just try one thing to make sure handle is good container.query() except DaosApiError as excep: print(excep) print(traceback.format_exc()) raise return
def setUp(self): super(ObjOpenBadParam, self).setUp() try: # parameters used in pool create createmode = self.params.get("mode", '/run/pool/createmode/') createsetid = self.params.get("setname", '/run/pool/createset/') createsize = self.params.get("size", '/run/pool/createsize/') createuid = os.geteuid() creategid = os.getegid() # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) # need a connection to create container self.pool.connect(1 << 1) # create a container self.container = DaosContainer(self.context) self.container.create(self.pool.handle) # now open it self.container.open() # create an object and write some data into it thedata = "a string that I want to stuff into an object" self.datasize = len(thedata) + 1 self.dkey = "this is the dkey" self.akey = "this is the akey" self.obj, self.epoch = self.container.write_an_obj(thedata, self.datasize, self.dkey, self.akey, obj_cls=1) thedata2 = self.container.read_an_obj(self.datasize, self.dkey, self.akey, self.obj, self.epoch) if thedata not in thedata2.value: print(thedata) print(thedata2.value) err_str = "Error reading back data, test failed during the " \ "initial setup." self.d_log.error(err_str) self.fail(err_str) # setup leaves object in open state, so closing to start clean self.obj.close() except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test failed during the initial setup.")
def create(self, uuid=None): """Create a container. Args: uuid (str, optional): contianer uuid. Defaults to None. """ self.destroy() self.log.info("Creating a container") self.container = DaosContainer(self.pool.context) self.container.create(self.pool.pool.handle, uuid) self.uuid = self.container.get_uuid_str()
def setUp(self): try: # get paths from the build_vars generated by build with open('../../../.build_vars.json') as f: build_paths = json.load(f) self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../") self.tmp = build_paths['PREFIX'] + '/tmp' self.server_group = self.params.get("server_group", '/server/', 'daos_server') # setup the DAOS python API self.Context = DaosContext(build_paths['PREFIX'] + '/lib/') self.hostlist = self.params.get("test_machines", '/run/hosts/*') self.hostfile = WriteHostFile.WriteHostFile( self.hostlist, self.tmp) ServerUtils.runServer(self.hostfile, self.server_group, self.basepath) # parameters used in pool create createmode = self.params.get("mode", '/run/pool/createmode/') createsetid = self.params.get("setname", '/run/pool/createset/') createsize = self.params.get("size", '/run/pool/createsize/') createuid = os.geteuid() creategid = os.getegid() # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.Context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) self.pool.connect(1 << 1) # create a container self.container = DaosContainer(self.Context) self.container.create(self.pool.handle) # now open it self.container.open() except DaosApiError as e: print(e) print(traceback.format_exc()) self.fail("Test failed during setup.\n")
def setUp(self): self.agent_sessions = None self.pool = None self.container = None self.obj = None self.ioreq = None self.hostlist = None self.hostfile = None self.no_of_dkeys = None self.no_of_akeys = None self.array_size = None self.record_length = None with open('../../../.build_vars.json') as json_f: build_paths = json.load(json_f) basepath = os.path.normpath(build_paths['PREFIX'] + "/../") server_group = self.params.get("name", '/server_config/', 'daos_server') self.context = DaosContext(build_paths['PREFIX'] + '/lib/') self.d_log = DaosLog(self.context) self.hostlist = self.params.get("test_machines", '/run/hosts/*') self.hostfile = write_host_file.write_host_file( self.hostlist, self.workdir) self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0] self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0] self.array_size = self.params.get("size", '/array_size/') self.record_length = self.params.get("length", '/run/record/*') self.agent_sessions = agent_utils.run_agent(basepath, self.hostlist) server_utils.run_server(self.hostfile, server_group, basepath) self.pool = DaosPool(self.context) self.pool.create(self.params.get("mode", '/run/pool/createmode/*'), os.geteuid(), os.getegid(), self.params.get("size", '/run/pool/createsize/*'), self.params.get("setname", '/run/pool/createset/*'), None) self.pool.connect(2) self.container = DaosContainer(self.context) self.container.create(self.pool.handle) self.container.open() self.obj = DaosObj(self.context, self.container) self.obj.create(objcls=1) self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4)
def test_global_handle(self): """ Test ID: DAO Test Description: Use a pool handle in another process. :avocado: tags=all,pool,pr,tiny,poolglobalhandle """ try: # use the uid/gid of the user running the test, these should # be perfectly valid createuid = os.geteuid() creategid = os.getegid() # parameters used in pool create that are in yaml createmode = self.params.get("mode", '/run/testparams/createmode/') createsetid = self.params.get("setname", '/run/testparams/createset/') createsize = self.params.get("size", '/run/testparams/createsize/') # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) pool.connect(1 << 1) # create a container just to make sure handle is good container = DaosContainer(self.context) container.create(pool.handle) # create a global handle iov_len, buf_len, buf = pool.local2global() # this should work in the future but need on-line server addition #arg_list = (buf_len, iov_len, buf, pool.get_uuid_str(), 0) #p = Process(target=check_handle, args=arg_list) #p.start() #p.join() # for now verifying global handle in the same process which is not # the intended use case check_handle(buf_len, iov_len, buf, pool.get_uuid_str(), 0) except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Expecting to pass but test has failed.\n")
def setUp(self): # get paths from the build_vars generated by build with open('../../../.build_vars.json') as finput: build_paths = json.load(finput) basepath = os.path.normpath(build_paths['PREFIX'] + "/../") server_group = self.params.get("name", '/server_config/', 'daos_server') # setup the DAOS python API self.context = DaosContext(build_paths['PREFIX'] + '/lib/') self.d_log = DaosLog(self.context) self.hostlist = self.params.get("test_machines", '/run/hosts/*') hostfile = write_host_file.write_host_file(self.hostlist, self.workdir) server_utils.run_server(hostfile, server_group, basepath) # Set up the pool and container. try: # parameters used in pool create createmode = self.params.get("mode", '/run/pool/createmode/') createsetid = self.params.get("setname", '/run/pool/createset/') createsize = self.params.get("size", '/run/pool/createsize/*') createuid = os.geteuid() creategid = os.getegid() # initialize a pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) # need a connection to create container self.pool.connect(1 << 1) # create a container self.container = DaosContainer(self.context) self.container.create(self.pool.handle) # now open it self.container.open() except DaosApiError as error: print(error) print(traceback.format_exc()) self.fail("Test failed before snapshot taken")
def setUp(self): super(ContainerAttributeTest, self).setUp() self.large_data_set = {} self.pool = DaosPool(self.context) self.pool.create( self.params.get("mode", '/run/attrtests/createmode/*'), os.geteuid(), os.getegid(), self.params.get("size", '/run/attrtests/createsize/*'), self.params.get("setname", '/run/attrtests/createset/*'), None) self.pool.connect(1 << 1) poh = self.pool.handle self.container = DaosContainer(self.context) self.container.create(poh) self.container.open()
def create(self, uuid=None): """Create a container. Args: uuid (str, optional): contianer uuid. Defaults to None. """ self.destroy() self.log.info("Creating a container with pool handle %s", self.pool.pool.handle.value) self.container = DaosContainer(self.pool.context) kwargs = {"poh": self.pool.pool.handle} if uuid is not None: kwargs["con_uuid"] = uuid self._call_method(self.container.create, kwargs) self.uuid = self.container.get_uuid_str() self.log.info(" Container created with uuid {}".format(self.uuid))
def setUp(self): """ set up method """ super(Snapshot, self).setUp() # get parameters from yaml file createmode = self.params.get("mode", '/run/poolparams/createmode/') createuid = os.geteuid() creategid = os.getegid() createsetid = self.params.get("setname", '/run/poolparams/createset/') createsize = self.params.get("size", '/run/poolparams/createsize/') self.log.info("==In setUp, self.context= %s", self.context) try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) # need a connection to the pool with rw permission # DAOS_PC_RO = int(1 << 0) # DAOS_PC_RW = int(1 << 1) # DAOS_PC_EX = int(1 << 2) self.pool.connect(1 << 1) # create a container self.container = DaosContainer(self.context) self.container.create(self.pool.handle) except DaosApiError as error: self.log.info("Error detected in DAOS pool container setup: %s", str(error)) self.log.info(traceback.format_exc()) self.fail("##Test failed on setUp, before snapshot taken") # now open it self.container.open() # do a query and compare the UUID returned from create with # that returned by query self.container.query() if self.container.get_uuid_str() != c_uuid_to_str( self.container.info.ci_uuid): self.fail("##Container UUID did not match the one in info.")
def test_metadata_addremove(self): """ Test ID: DAOS-1512 Test Description: Verify metadata release the space after container delete. :avocado: tags=metadata,metadata_free_space,nvme,small """ self.pool.connect(2) for k in range(10): container_array = [] self.d_log.debug("Container Create Iteration {}".format(k)) for cont in range(NO_OF_MAX_CONTAINER): container = DaosContainer(self.context) container.create(self.pool.handle) container_array.append(container) self.d_log.debug("Container Remove Iteration {} ".format(k)) for cont in container_array: cont.destroy()
def get_container(context, pool, log=None): """Retrun a DAOS a container that has been created an opened. Args: context (DaosContext): the context to use to create the container pool (DaosPool): pool in which to create the container log (DaosLog|None): object for logging messages Returns: DaosContainer: an object representing a DAOS container """ if log: log.info("Creating a container") container = DaosContainer(context) container.create(pool.handle) if log: log.info("Opening a container") container.open() return container
def test_container_basics(self): """ Test basic container create/destroy/open/close/query. Nothing fancy just making sure they work at a rudimentary level :avocado: tags=container,containercreate,containerdestroy,basecont """ pool = None hostlist = None try: hostlist = self.params.get("test_machines", '/run/hosts/*') hostfile = write_host_file.write_host_file(hostlist, self.workdir) self.agent_sessions = agent_utils.run_agent(self.basepath, hostlist) server_utils.run_server(hostfile, self.server_group, self.basepath) # give it time to start time.sleep(2) # parameters used in pool create createmode = self.params.get("mode", '/run/conttests/createmode/') createuid = self.params.get("uid", '/run/conttests/createuid/') creategid = self.params.get("gid", '/run/conttests/creategid/') createsetid = self.params.get("setname", '/run/conttests/createset/') createsize = self.params.get("size", '/run/conttests/createsize/') # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) # need a connection to create container pool.connect(1 << 1) # create a container container = DaosContainer(self.context) container.create(pool.handle) # now open it container.open() # do a query and compare the UUID returned from create with # that returned by query container.query() if container.get_uuid_str() != c_uuid_to_str( container.info.ci_uuid): self.fail("Container UUID did not match the one in info'n") container.close() # wait a few seconds and then destroy time.sleep(5) container.destroy() except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test was expected to pass but it failed.\n") except Exception as excep: self.fail("Daos code segfaulted most likely, error: %s" % excep) finally: # cleanup the pool if pool is not None: pool.disconnect() pool.destroy(1) if self.agent_sessions: agent_utils.stop_agent(self.agent_sessions) server_utils.stop_server(hosts=hostlist)
def test_no_space_cont_create(self): """ :avocado: tags=all,container,tiny,full_regression,fullpoolcontcreate """ # full storage rc err = "-1007" # probably should be -1007, revisit later err2 = "-1009" # create pool self.pool = DaosPool(self.context) mode = self.params.get("mode", '/conttests/createmode/') self.d_log.debug("mode is {0}".format(mode)) uid = os.geteuid() gid = os.getegid() # 16 mb pool, minimum size currently possible size = 16777216 self.d_log.debug("creating pool") self.pool.create(mode, uid, gid, size, self.server_group, None) self.d_log.debug("created pool") # connect to the pool self.d_log.debug("connecting to pool") self.pool.connect(1 << 1) self.d_log.debug("connected to pool") # query the pool self.d_log.debug("querying pool info") dummy_pool_info = self.pool.pool_query() self.d_log.debug("queried pool info") # create a container try: self.d_log.debug("creating container") self.cont = DaosContainer(self.context) self.cont.create(self.pool.handle) self.d_log.debug("created container") except DaosApiError as excep: self.d_log.error("caught exception creating container: " "{0}".format(excep)) self.fail("caught exception creating container: {0}".format(excep)) self.d_log.debug("opening container") self.cont.open() self.d_log.debug("opened container") # generate random dkey, akey each time # write 1mb until no space, then 1kb, etc. to fill pool quickly for obj_sz in [1048576, 10240, 10, 1]: write_count = 0 while True: self.d_log.debug("writing obj {0}, sz {1} to " "container".format(write_count, obj_sz)) my_str = "a" * obj_sz my_str_sz = obj_sz dkey = (''.join( random.choice(string.lowercase) for i in range(5))) akey = (''.join( random.choice(string.lowercase) for i in range(5))) try: dummy_oid, dummy_tx = self.cont.write_an_obj( my_str, my_str_sz, dkey, akey, obj_cls="OC_SX") self.d_log.debug("wrote obj {0}, sz {1}".format( write_count, obj_sz)) write_count += 1 except DaosApiError as excep: if not (err in repr(excep) or err2 in repr(excep)): self.d_log.error("caught exception while writing " "object: {0}".format(repr(excep))) self.fail("caught exception while writing object: {0}". format(repr(excep))) else: self.d_log.debug("pool is too full for {0} byte " "objects".format(obj_sz)) break self.d_log.debug("closing container") self.cont.close() self.d_log.debug("closed container") # create a 2nd container now that pool is full try: self.d_log.debug("creating 2nd container") self.cont2 = DaosContainer(self.context) self.cont2.create(self.pool.handle) self.d_log.debug("created 2nd container") self.d_log.debug("opening container 2") self.cont2.open() self.d_log.debug("opened container 2") self.d_log.debug("writing one more object, write expected to fail") self.cont2.write_an_obj(my_str, my_str_sz, dkey, akey, obj_cls="OC_SX") self.fail("wrote one more object after pool was completely filled," " this should never print") except DaosApiError as excep: if not (err in repr(excep) or err2 in repr(excep)): self.d_log.error("caught unexpected exception while " "writing object: {0}".format(repr(excep))) self.fail("caught unexpected exception while writing " "object: {0}".format(repr(excep))) else: self.d_log.debug("correctly caught -1007 while attempting " "to write object in full pool")
def test_closehandle(self): """ Test container close function with container handle paramter. :avocado: tags=container,openclose,closehandle """ saved_coh = None # parameters used in pool create createmode = self.params.get("mode", '/run/pool/createmode/') createuid = os.geteuid() creategid = os.getegid() createsetid = self.params.get("setname", '/run/pool/createset/') createsize = self.params.get("size", '/run/pool/createsize/') coh_params = self.params.get("coh", '/run/container/container_handle/*/') expected_result = coh_params[1] try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.Context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.Container1 = DaosContainer(self.Context) self.Container1.create(poh) str_cuuid = self.Container1.get_uuid_str() cuuid = uuid.UUID(str_cuuid) self.Container1.open(poh, cuuid, 2, None) # Defining 'good' and 'bad' container handles saved_coh = self.Container1.coh if coh_params[0] == 'GOOD': coh = self.Container1.coh else: # create a second container, open to get a handle # then close & destroy so handle is invalid self.Container2 = DaosContainer(self.Context) self.Container2.create(poh) self.Container2.open(poh, cuuid, 2, None) coh = self.Container2.coh self.Container2.close() self.Container2.destroy() # close container with either good or bad handle self.Container1.close(coh) if expected_result in ['FAIL']: self.fail("Test was expected to fail but it passed.\n") except DaosApiError as e: if expected_result == 'PASS': print(e) print(traceback.format_exc()) self.fail("Test was expected to pass but it failed.\n") # close above failed so close for real with the right coh if saved_coh is not None: self.Container1.close(saved_coh) finally: self.Container1.destroy(1) self.pool.disconnect() self.pool.destroy(1) self.pool = None
def test_container_open(self): """ Test basic container bad create. :avocado: tags=container,containeropen """ expected_for_param = [] uuidlist = self.params.get("uuid",'/run/createtests/uuids/*/') containerUUID = uuidlist[0] expected_for_param.append(uuidlist[1]) pohlist = self.params.get("poh",'/run/createtests/handles/*/') poh = pohlist[0] expected_for_param.append(pohlist[1]) expected_result = 'PASS' for result in expected_for_param: if result == 'FAIL': expected_result = 'FAIL' break try: # create two pools and try to create containers in these pools self.POOL1 = DaosPool(self.Context) self.POOL1.create(self.createmode, self.createuid1, self.creategid1, self.createsize, self.createsetid, None) self.POOL2 = DaosPool(self.Context) self.POOL2.create(self.createmode, self.createuid2, self.creategid2, self.createsize, None, None) # Connect to the pools self.POOL1.connect(1 << 1) self.POOL2.connect(1 << 1) # defines pool handle for container open if pohlist[0] == 'POOL1': poh = self.POOL1.handle else: poh = self.POOL2.handle # Create a container in POOL1 self.CONTAINER1 = DaosContainer(self.Context) self.CONTAINER1.create(self.POOL1.handle) # defines test UUID for container open if uuidlist[0] == 'POOL1': struuid = self.CONTAINER1.get_uuid_str() containerUUID = uuid.UUID(struuid) else: if uuidlist[0] == 'MFUUID': containerUUID = "misformed-uuid-0000" else: containerUUID = uuid.uuid4() # random uuid # tries to open the container1 # open should be ok only if poh = POOL1.handle && containerUUID = CONTAINER1.uuid self.CONTAINER1.open(poh, containerUUID) # wait a few seconds and then destroy containers time.sleep(5) self.CONTAINER1.close() self.CONTAINER1.destroy() self.CONTAINER1 = None # cleanup the pools self.POOL1.disconnect() self.POOL1.destroy(1) self.POOL1 = None self.POOL2.disconnect() self.POOL2.destroy(1) self.POOL2 = None if expected_result in ['FAIL']: self.fail("Test was expected to fail but it passed.\n") except DaosApiError as e: print(e) print(traceback.format_exc()) if expected_result == 'PASS': self.fail("Test was expected to pass but it failed.\n") finally: if self.hostfile is not None: os.remove(self.hostfile)
def test_queryasync(self): """ Test container query for asynchronous mode. :avocado: tags=container,containerasync,queryasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode",'/run/createtests/createmode/*/') createsetid = self.params.get("setname",'/run/createtests/createset/') createsize = self.params.get("size",'/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.POOL = DaosPool(self.Context) self.POOL.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.POOL.handle self.POOL.connect(1 << 1) # Container initialization and creation self.Container1 = DaosContainer(self.Context) self.Container2 = DaosContainer(self.Context) self.Container1.create(poh) str_cuuid = self.Container1.get_uuid_str() cuuid = uuid.UUID(str_cuuid) coh = self.Container1.coh # Open Container self.Container1.open(poh, None, 2, None, coh) GLOB_SIGNAL = threading.Event() self.Container1.query(coh, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test:{0}".format(GLOB_RC)) print ("RC after successful Container create: " , GLOB_RC) # Close opened Container self.Container1.close(coh) # Try to open container2, this should fail, as non-existent. # Checking rc after failure. GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.Container2.query(coh, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != -1002: self.fail("RC not as expected in async test:{0}".format(GLOB_RC)) print ("RC after Container destroy failed:", GLOB_RC) # cleanup the Pool and Container self.Container1.destroy() self.POOL.disconnect() self.POOL.destroy(1) self.POOL = None except ValueError as e: print e print traceback.format_exc()
def test_rebuild_with_io(self): """ Test ID: Rebuild-003 Test Description: Trigger a rebuild while I/O is ongoing. Use Cases: -- single pool, single client performing continous read/write/verify sequence while failure/rebuild is triggered in another process :avocado: tags=pool,rebuild,rebuildwithio """ # the rebuild tests need to redo this stuff each time so not in setup # as it usually would be server_group = self.params.get("name", '/server_config/', 'daos_server') self.hostlist_servers = self.params.get("test_machines", '/run/hosts/') hostfile_servers = write_host_file.write_host_file( self.hostlist_servers, self.workdir) try: self.agent_sessions = agent_utils.run_agent(self.basepath, self.hostlist_servers) server_utils.run_server(hostfile_servers, server_group, self.basepath) # use the uid/gid of the user running the test, these should # be perfectly valid createuid = os.geteuid() creategid = os.getegid() # parameters used in pool create that are in yaml createmode = self.params.get("mode", '/run/testparams/createmode/') createsetid = self.params.get("setname", '/run/testparams/createset/') createsize = self.params.get("size", '/run/testparams/createsize/') # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) pool.connect(1 << 1) container = DaosContainer(self.context) container.create(pool.handle) container.open() # get pool status and make sure it all looks good before we start pool.pool_query() if pool.pool_info.pi_ndisabled != 0: self.fail("Number of disabled targets reporting incorrectly.\n") if pool.pool_info.pi_rebuild_st.rs_errno != 0: self.fail("Rebuild error but rebuild hasn't run.\n") if pool.pool_info.pi_rebuild_st.rs_done != 1: self.fail("Rebuild is running but device hasn't failed yet.\n") if pool.pool_info.pi_rebuild_st.rs_obj_nr != 0: self.fail("Rebuilt objs not zero.\n") if pool.pool_info.pi_rebuild_st.rs_rec_nr != 0: self.fail("Rebuilt recs not zero.\n") dummy_pool_version = pool.pool_info.pi_rebuild_st.rs_version # do I/O for 30 seconds dummy_bw = io_utilities.continuous_io(container, 30) # trigger the rebuild rank = self.params.get("rank", '/run/testparams/ranks/*') server = DaosServer(self.context, server_group, rank) server.kill(1) pool.exclude([rank]) # do another 30 seconds of I/O, # waiting for some improvements in server bootstrap # at which point we can move the I/O to a separate client and # really pound it with I/O dummy_bw = io_utilities.continuous_io(container, 30) # wait for the rebuild to finish while True: pool.pool_query() if pool.pool_info.pi_rebuild_st.rs_done == 1: break else: time.sleep(2) # check rebuild statistics if pool.pool_info.pi_ndisabled != 1: self.fail("Number of disabled targets reporting incorrectly: {}" .format(pool.pool_info.pi_ndisabled)) if pool.pool_info.pi_rebuild_st.rs_errno != 0: self.fail("Rebuild error reported: {}".format( pool.pool_info.pi_rebuild_st.rs_errno)) if pool.pool_info.pi_rebuild_st.rs_obj_nr <= 0: self.fail("No objects have been rebuilt.") if pool.pool_info.pi_rebuild_st.rs_rec_nr <= 0: self.fail("No records have been rebuilt.") except (ValueError, DaosApiError) as excep: print(excep) print(traceback.format_exc()) self.fail("Expecting to pass but test has failed.\n") finally: # wait for the I/O process to finish try: server_utils.stop_server(hosts=self.hostlist_servers) os.remove(hostfile_servers) # really make sure everything is gone check_for_pool.cleanup_pools(self.hostlist_servers) finally: if self.agent_sessions: agent_utils.stop_agent(self.agent_sessions) server_utils.kill_server(self.hostlist_servers)
def test_closeasync(self): """ Test container close for asynchronous mode. :avocado: tags=all,small,full_regression,container,closeasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) self.container1.create(poh) str_cuuid = self.container1.get_uuid_str() cuuid = uuid.UUID(str_cuuid) self.container1.open(poh, cuuid, 2) GLOB_SIGNAL = threading.Event() self.container1.close(cb_func=cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test: " "{0}".format(GLOB_RC)) print("RC after successful container create: ", GLOB_RC) # Try to open container2, this should fail, as non-existent. # Checking rc after failure. GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.close(cb_func=cb_func) GLOB_SIGNAL.wait() if GLOB_RC == 0: self.fail("RC not as expected in async test: " "{0}".format(GLOB_RC)) print("RC after container destroy failed:", GLOB_RC) # cleanup the pool and container self.container1.destroy() self.pool.disconnect() self.pool.destroy(1) self.pool = None except DaosApiError as excep: print(excep) print(traceback.format_exc())
def test_global_handle(self): """ Test ID: DAO Test Description: Use a pool handle in another process. :avocado: tags=container,conthandle,vm,small,regression """ try: # use the uid/gid of the user running the test, these should # be perfectly valid createuid = os.geteuid() creategid = os.getegid() # parameters used in pool create that are in yaml createmode = self.params.get("mode", '/run/testparams/createmode/') createsetid = self.params.get("setname", '/run/testparams/createset/') createsize = self.params.get("size", '/run/testparams/createsize/') # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.Context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) pool.connect(1 << 1) # create a pool global handle iov_len, buf_len, buf = pool.local2global() buftype = ctypes.c_byte * buf_len c_buf = buftype.from_buffer(buf) sct_pool_handle = sharedctypes.RawValue( IOV, ctypes.cast(c_buf, ctypes.c_void_p), buf_len, iov_len) # create a container container = DaosContainer(self.Context) container.create(pool.handle) container.open() # create a container global handle iov_len, buf_len, buf = container.local2global() buftype = ctypes.c_byte * buf_len c_buf = buftype.from_buffer(buf) sct_cont_handle = sharedctypes.RawValue( IOV, ctypes.cast(c_buf, ctypes.c_void_p), buf_len, iov_len) sct_pool_uuid = sharedctypes.RawArray(ctypes.c_byte, pool.uuid) # this should work in the future but need on-line server addition #arg_list = ( #p = Process(target=CheckHandle, args=arg_list) #p.start() #p.join() # for now verifying global handle in the same process which is not # the intended use case CheckHandle(sct_pool_handle, sct_pool_uuid, sct_cont_handle, 0) except DaosApiError as e: print(e) print(traceback.format_exc()) self.fail("Expecting to pass but test has failed.\n")
def test_multipool_rebuild(self): """ Test ID: Rebuild-002 Test Description: Expand on the basic test by rebuilding 2 pools at once. Use Cases: -- multipool rebuild, single client, various object and record counds :avocado: tags=pool,rebuild,rebuildmulti """ # the rebuild tests need to redo this stuff each time so not in setup # as it usually would be setid = self.params.get("setname", '/run/testparams/setnames/') server_group = self.params.get("server_group", '/server/', 'daos_server') basepath = os.path.normpath(self.build_paths['PREFIX'] + "/../") tmp = self.build_paths['PREFIX'] + '/tmp' self.hostlist = self.params.get("test_machines", '/run/hosts/') hostfile = WriteHostFile.WriteHostFile(self.hostlist, tmp) try: ServerUtils.runServer(hostfile, server_group, basepath) # use the uid/gid of the user running the test, these should # be perfectly valid createuid = os.geteuid() creategid = os.getegid() # parameters used in pool create that are in yaml createmode = self.params.get("mode", '/run/testparams/createmode/') createsetid = self.params.get("setname", '/run/testparams/createset/') createsize = self.params.get("size", '/run/testparams/createsize/') # initialize python pool object then create the underlying # daos storage, the way the code is now the pools should be # on the same storage and have the same service leader pool1 = DaosPool(self.Context) pool2 = DaosPool(self.Context) pool1.create(createmode, createuid, creategid, createsize, createsetid, None) pool2.create(createmode, createuid, creategid, createsize, createsetid, None) # want an open connection during rebuild pool1.connect(1 << 1) pool2.connect(1 << 1) # create containers container1 = DaosContainer(self.Context) container1.create(pool1.handle) container2 = DaosContainer(self.Context) container2.create(pool2.handle) # now open them container1.open() container2.open() # how many objects and records are we creating objcount = self.params.get("objcount", '/run/testparams/numobjects/*') reccount = self.params.get("reccount", '/run/testparams/numrecords/*') if objcount == 0: reccount = 0 # which rank to write to and kill rank = self.params.get("rank", '/run/testparams/ranks/*') # how much data to write with each key size = self.params.get("size", '/run/testparams/datasize/') # Putting the same data in both pools, at least for now to simplify # checking its correct saved_data = [] for i in range(0, objcount): obj = None for j in range(0, reccount): # make some stuff up and write dkey = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(5)) akey = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(5)) data = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) obj, tx = container1.write_an_obj(data, len(data), dkey, akey, obj, rank) obj, tx = container2.write_an_obj(data, len(data), dkey, akey, obj, rank) saved_data.append((obj, dkey, akey, data, tx)) # read the data back and make sure its correct # containers data2 = container1.read_an_obj(size, dkey, akey, obj, tx) if data != data2.value: self.fail( "Wrote data P1, read it back, didn't match\n") # containers data2 = container2.read_an_obj(size, dkey, akey, obj, tx) if data != data2.value: self.fail( "Wrote data P2, read it back, didn't match\n") # kill a server server = DaosServer(self.Context, server_group, rank) server.kill(1) # temporarily, the exclude of a failed target must be done # manually pool1.exclude([rank]) pool2.exclude([rank]) # check that rebuild finishes, no errors, progress data as # know it to be. Check pool 1 first then we'll check 2 below. while True: pool1.pool_query() if pool1.pool_info.pi_rebuild_st.rs_done == 1: break else: time.sleep(2) # check there are no errors and other data matches what we # apriori know to be true, if pool1.pool_info.pi_ndisabled != 1: self.fail( "P1 number disabled targets reporting incorrectly: {}". format(pool1.pool_info.pi_ndisabled)) if pool1.pool_info.pi_rebuild_st.rs_errno != 0: self.fail("P1 rebuild error reported: {}".format( pool1.pool_info.pi_rebuild_st.rs_errno)) if pool1.pool_info.pi_rebuild_st.rs_obj_nr != objcount: self.fail("P1 rebuilt objs not as expected: {0} {1}".format( pool1.pool_info.pi_rebuild_st.rs_obj_nr, objcount)) if pool1.pool_info.pi_rebuild_st.rs_rec_nr != (reccount * objcount): self.fail("P1 rebuilt recs not as expected: {0} {1}".format( pool1.pool_info.pi_rebuild_st.rs_rec_nr, reccount * objcount)) # now that the rebuild finished verify the records are correct for tup in saved_data: data2 = container1.read_an_obj(len(tup[3]), tup[1], tup[2], tup[0], tup[4]) if tup[3] != data2.value: self.fail("after rebuild data didn't check out") # now check the other pool while True: pool2.pool_query() if pool2.pool_info.pi_rebuild_st.rs_done == 1: break else: time.sleep(2) # check there are no errors and other data matches what we # apriori know to be true if pool2.pool_info.pi_ndisabled != 1: self.fail( "Number disabled targets reporting incorrectly: {}".format( pool2.pool_info.pi_ndisabled)) if pool2.pool_info.pi_rebuild_st.rs_errno != 0: self.fail("Rebuild error reported: {}".format( pool2.pool_info.pi_rebuild_st.rs_errno)) if pool2.pool_info.pi_rebuild_st.rs_obj_nr != objcount: self.fail("Rebuilt objs not as expected: {0} {1}".format( pool2.pool_info.pi_rebuild_st.rs_obj_nr, objcount)) if pool2.pool_info.pi_rebuild_st.rs_rec_nr != (reccount * objcount): self.fail("Rebuilt recs not as expected: {0} {1}".format( pool2.pool_info.pi_rebuild_st.rs_rec_nr, (reccount * objcount))) # now that the rebuild finished verify the records are correct for tup in saved_data: data2 = container2.read_an_obj(len(tup[3]), tup[1], tup[2], tup[0], tup[4]) if tup[3] != data2.value: self.fail("after rebuild data didn't check out") except DaosApiError as e: print(e) print(traceback.format_exc()) self.fail("Expecting to pass but test has failed.\n") finally: ServerUtils.stopServer(hosts=self.hostlist) os.remove(hostfile) CheckForPool.CleanupPools(self.hostlist) ServerUtils.killServer(self.hostlist)
def test_array_obj(self): """ Test ID: DAOS-961 Test Description: Writes an array to an object and then reads it back and verifies it. :avocado: tags=object,arrayobj,regression,vm,small """ try: # parameters used in pool create createmode = self.params.get("mode", '/run/conttests/createmode/') createsetid = self.params.get("setname", '/run/conttests/createset/') createsize = self.params.get("size", '/run/conttests/createsize/') createuid = os.geteuid() creategid = os.getegid() print("uid is {} gid is {}".format(createuid, creategid)) # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.Context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) self.pl.info("Pool %s created.", pool.get_uuid_str()) # need a connection to create container pool.connect(1 << 1) # create a container container = DaosContainer(self.Context) container.create(pool.handle) self.pl.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # do a query and compare the UUID returned from create with # that returned by query container.query() if container.get_uuid_str() != c_uuid_to_str( container.info.ci_uuid): self.fail("Container UUID did not match the one in info\n") # create an object and write some data into it thedata = [] thedata.append("data string one") thedata.append("data string two") thedata.append("data string tre") dkey = "this is the dkey" akey = "this is the akey" self.pl.info("writing array to dkey >%s< akey >%s<.", dkey, akey) oid, epoch = container.write_an_array_value(thedata, dkey, akey) # read the data back and make sure its correct length = len(thedata[0]) thedata2 = container.read_an_array(len(thedata), length + 1, dkey, akey, oid, epoch) if thedata[0][0:length - 1] != thedata2[0][0:length - 1]: self.pl.error("Data mismatch") self.pl.error("Wrote: >%s<" (thedata[0])) self.pl.error("Read: >%s<" (thedata2[0])) self.fail("Write data, read it back, didn't match\n") if thedata[2][0:length - 1] != thedata2[2][0:length - 1]: self.pl.error("Data mismatch") self.pl.error("Wrote: >%s<" (thedata[2])) self.pl.error("Read: >%s<" (thedata2[2])) self.fail("Write data, read it back, didn't match\n") container.close() # wait a few seconds and then destroy time.sleep(5) container.destroy() # cleanup the pool pool.disconnect() pool.destroy(1) self.pl.info("Test Complete") except ValueError as e: self.pl.error("Test Failed, exception was thrown.") print e print traceback.format_exc() self.fail("Test was expected to pass but it failed.\n")
def test_simple_rebuild(self): """ Test ID: Rebuild-001 Test Description: The most basic rebuild test. Use Cases: -- single pool rebuild, single client, various reord/object counts :avocado: tags=pool,rebuild,rebuildsimple """ # the rebuild tests need to redo this stuff each time so not in setup # as it usually would be setid = self.params.get("setname", '/run/testparams/setnames/') server_group = self.params.get("server_group", '/server/', 'daos_server') basepath = os.path.normpath(self.build_paths['PREFIX'] + "/../") tmp = self.build_paths['PREFIX'] + '/tmp' self.hostlist = self.params.get("test_machines", '/run/hosts/') hostfile = WriteHostFile.WriteHostFile(self.hostlist, tmp) try: ServerUtils.runServer(hostfile, server_group, basepath) # use the uid/gid of the user running the test, these should # be perfectly valid createuid = os.geteuid() creategid = os.getegid() # parameters used in pool create that are in yaml createmode = self.params.get("mode", '/run/testparams/createmode/') createsetid = self.params.get("setname", '/run/testparams/createset/') createsize = self.params.get("size", '/run/testparams/createsize/') # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.Context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) # want an open connection during rebuild pool.connect(1 << 1) # get pool status we want to test later pool.pool_query() if pool.pool_info.pi_ndisabled != 0: self.fail( "Number of disabled targets reporting incorrectly.\n") if pool.pool_info.pi_rebuild_st.rs_errno != 0: self.fail("Rebuild error but rebuild hasn't run.\n") if pool.pool_info.pi_rebuild_st.rs_done != 1: self.fail("Rebuild is running but device hasn't failed yet.\n") if pool.pool_info.pi_rebuild_st.rs_obj_nr != 0: self.fail("Rebuilt objs not zero.\n") if pool.pool_info.pi_rebuild_st.rs_rec_nr != 0: self.fail("Rebuilt recs not zero.\n") pool_version = pool.pool_info.pi_rebuild_st.rs_version # create a container container = DaosContainer(self.Context) container.create(pool.handle) # now open it container.open() # how many objects and records are we creating objcount = self.params.get("objcount", '/run/testparams/numobjects/*') reccount = self.params.get("reccount", '/run/testparams/numrecords/*') if objcount == 0: reccount = 0 # which rank to write to and kill rank = self.params.get("rank", '/run/testparams/ranks/*') # how much data to write with each key size = self.params.get("size", '/run/testparams/datasize/') saved_data = [] for i in range(0, objcount): obj = None for j in range(0, reccount): # make some stuff up and write dkey = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(5)) akey = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(5)) data = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) obj, tx = container.write_an_obj(data, len(data), dkey, akey, obj, rank) saved_data.append((obj, dkey, akey, data, tx)) # read the data back and make sure its correct data2 = container.read_an_obj(size, dkey, akey, obj, tx) if data != data2.value: self.fail("Write data 1, read it back, didn't match\n") # kill a server that has server = DaosServer(self.Context, server_group, rank) server.kill(1) # temporarily, the exclude of a failed target must be done # manually pool.exclude([rank]) while True: # get the pool/rebuild status again pool.pool_query() if pool.pool_info.pi_rebuild_st.rs_done == 1: break else: time.sleep(2) if pool.pool_info.pi_ndisabled != 1: self.fail( "Number of disabled targets reporting incorrectly: {}". format(pool.pool_info.pi_ndisabled)) if pool.pool_info.pi_rebuild_st.rs_errno != 0: self.fail("Rebuild error reported: {}".format( pool.pool_info.pi_rebuild_st.rs_errno)) if pool.pool_info.pi_rebuild_st.rs_obj_nr != objcount: self.fail("Rebuilt objs not as expected: {0} {1}".format( pool.pool_info.pi_rebuild_st.rs_obj_nr, objcount)) if pool.pool_info.pi_rebuild_st.rs_rec_nr != (reccount * objcount): self.fail("Rebuilt recs not as expected: {0} {1}".format( pool.pool_info.pi_rebuild_st.rs_rec_nr, reccount * objcount)) # now that the rebuild finished verify the records are correct for tup in saved_data: data2 = container.read_an_obj(len(tup[3]), tup[1], tup[2], tup[0], tup[4]) if tup[3] != data2.value: self.fail("after rebuild data didn't check out") except DaosApiError as e: print(e) print(traceback.format_exc()) self.fail("Expecting to pass but test has failed.\n") finally: try: ServerUtils.stopServer(hosts=self.hostlist) os.remove(hostfile) # really make sure everything is gone CheckForPool.CleanupPools(self.hostlist) finally: ServerUtils.killServer(self.hostlist)