def test_bad_handle(self): """ Test ID: DAOS-1376 Test Description: Pass a bogus object handle, should return bad handle. :avocado: tags=all,object,full_regression,small,objbadhand """ self.prepare_pool() try: # create a container container = DaosContainer(self.context) container.create(self.pool.pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # create an object and write some data into it thedata = "a string that I want to stuff into an object" thedatasize = len(thedata) + 1 dkey = "this is the dkey" akey = "this is the akey" obj = container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) saved_oh = obj.obj_handle obj.obj_handle = 99999 obj = container.write_an_obj(thedata, thedatasize, dkey, akey, obj, None, 2) container.oh = saved_oh container.close() container.destroy() self.fail("Test was expected to return a -1002 but it has not.\n") except DaosApiError as excep: container.oh = saved_oh container.close() container.destroy() self.plog.info("Test Complete") if '-1002' not in str(excep): print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1002 but it has not.\n")
class OSAUtils(MdtestBase, IorTestBase): # pylint: disable=too-many-ancestors """ Test Class Description: This test runs daos_server offline drain test cases. :avocado: recursive """ def setUp(self): """Set up for test case.""" super().setUp() self.pool_cont_dict = {} self.container = None self.obj = None self.ioreq = None self.dmg_command = self.get_dmg_command() self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*', default=[0])[0] self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*', default=[0])[0] self.record_length = self.params.get("length", '/run/record/*', default=[0])[0] self.ior_w_flags = self.params.get("write_flags", '/run/ior/iorflags/*', default="") self.ior_r_flags = self.params.get("read_flags", '/run/ior/iorflags/*') self.server_count = len(self.hostlist_servers) self.engine_count = self.server_managers[0].get_config_value( "engines_per_host") self.out_queue = queue.Queue() self.dmg_command.exit_status_exception = False self.test_during_aggregation = False self.test_during_rebuild = False self.test_with_checksum = True # By default, test_with_rf is set to False. # It is up to individual test to enable it. self.test_with_rf = False self.test_with_blank_node = False self.test_with_snapshot = False @fail_on(CommandFailure) def get_pool_leader(self): """Get the pool leader. Returns: int: pool leader value """ data = self.dmg_command.pool_query(self.pool.uuid) return int(data["response"]["leader"]) @fail_on(CommandFailure) def get_rebuild_status(self): """Get the rebuild status. Returns: str: rebuild status """ data = self.dmg_command.pool_query(self.pool.uuid) return data["response"]["rebuild"]["status"] @fail_on(CommandFailure) def get_rebuild_state(self): """Get the rebuild state. Returns: str: rebuild state """ data = self.dmg_command.pool_query(self.pool.uuid) return data["response"]["rebuild"]["state"] @fail_on(CommandFailure) def is_rebuild_done(self, time_interval, wait_for_rebuild_to_complete=False): """Rebuild is completed/done. Args: time_interval: Wait interval between checks wait_for_rebuild_to_complete: Rebuild completed (Default: False) """ self.pool.wait_for_rebuild(wait_for_rebuild_to_complete, interval=time_interval) @fail_on(CommandFailure) def assert_on_rebuild_failure(self): """If the rebuild is not successful, raise assert. """ rebuild_status = self.get_rebuild_status() self.log.info("Rebuild Status: %s", rebuild_status) rebuild_failed_string = ["failed", "scanning", "aborted", "busy"] self.assertTrue(rebuild_status not in rebuild_failed_string, "Rebuild failed") @fail_on(CommandFailure) def print_and_assert_on_rebuild_failure(self, out, timeout=3): """Print the out value (daos, dmg, etc) and check for rebuild completion. If not, raise assert. """ self.log.info(out) self.is_rebuild_done(timeout) self.assert_on_rebuild_failure() @fail_on(CommandFailure) def get_pool_version(self): """Get the pool version. Returns: int: pool_version_value """ data = self.dmg_command.pool_query(self.pool.uuid) return int(data["response"]["version"]) @fail_on(CommandFailure) def get_ipaddr_for_rank(self, rank=None): """Obtain the IPAddress and port number for a particular server rank. Args: rank (int): daos_engine rank. Defaults to None. Returns: ip_addr (str) : IPAddress for the rank. port_num (str) : Port number for the rank. """ output = self.dmg_command.system_query() members_length = self.server_count * self.engine_count for i in range(0, members_length): if rank == int(output["response"]["members"][i]["rank"]): temp = output["response"]["members"][i]["addr"] ip_addr = temp.split(":") temp = output["response"]["members"][i]["fabric_uri"] port_num = temp.split(":") return ip_addr[0], port_num[2] return None, None @fail_on(CommandFailure) def remove_pool_dir(self, ip_addr=None, port_num=None): """Remove the /mnt/daos[x]/<pool_uuid>/vos-* directory Args: ip_addr (str): IP address of the daos server. Defaults to None. port_number (str) : Port number the daos server. """ # Create the expected port list # expected_ports = [port0] - Single engine/server # expected_ports = [port0, port1] - Two engine/server expected_ports = [engine_param.get_value("fabric_iface_port") for engine_param in self.server_managers[-1]. manager.job.yaml.engine_params] self.log.info("Expected ports : %s", expected_ports) if ip_addr is None or port_num is None: self.log.info("ip_addr : %s port_number: %s", ip_addr, port_num) self.fail("No IP Address or Port number provided") else: if self.engine_count == 1: self.log.info("Single Engine per Server") cmd = "/usr/bin/ssh {} -oStrictHostKeyChecking=no \ sudo rm -rf /mnt/daos/{}/vos-*". \ format(ip_addr, self.pool.uuid) elif self.engine_count == 2: if port_num == str(expected_ports[0]): port_val = 0 elif port_num == str(expected_ports[1]): port_val = 1 else: self.log.info("port_number: %s", port_num) self.fail("Invalid port number") cmd = "/usr/bin/ssh {} -oStrictHostKeyChecking=no \ sudo rm -rf /mnt/daos{}/{}/vos-*". \ format(ip_addr, port_val, self.pool.uuid) else: self.fail("Not supported engine per server configuration") run_command(cmd) def set_container(self, container): """Set the OSA utils container object. Args: container (obj) : Container object to be used within OSA utils. """ self.container = container def simple_osa_reintegrate_loop(self, rank, action="exclude", loop_time=100): """This method performs exclude or drain and reintegration on a rank for a certain amount of time. Args: rank (int): daos server rank. action (str) : "exclude" or "drain". Defaults to "exclude" loop_time: Total time to perform drain/reintegrate operation in a loop. (Default : 100 secs) """ start_time = 0 finish_time = 0 start_time = time.time() while int(finish_time - start_time) < loop_time: if action == "exclude": output = self.dmg_command.pool_exclude(self.pool.uuid, rank) else: output = self.dmg_command.pool_drain(self.pool.uuid, rank) self.print_and_assert_on_rebuild_failure(output) output = self.dmg_command.pool_reintegrate(self.pool.uuid, rank) self.print_and_assert_on_rebuild_failure(output) finish_time = time.time() @fail_on(DaosApiError) def write_single_object(self): """Write some data to the existing pool.""" self.pool.connect(2) csum = self.params.get("enable_checksum", '/run/container/*') self.container = DaosContainer(self.context) input_param = self.container.cont_input_values input_param.enable_chksum = csum self.container.create(poh=self.pool.pool.handle, con_prop=input_param) self.container.open() self.obj = DaosObj(self.context, self.container) self.obj.create(objcls=1) self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) self.log.info("Writing the Single Dataset") for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length) d_key_value = "dkey {0}".format(dkey) c_dkey = create_string_buffer(d_key_value) a_key_value = "akey {0}".format(akey) c_akey = create_string_buffer(a_key_value) c_value = create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size) self.obj.close() self.container.close() @fail_on(DaosApiError) def verify_single_object(self): """Verify the container data on the existing pool.""" self.pool.connect(2) self.container.open() self.obj.open() self.log.info("Single Dataset Verification -- Started") for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length) c_dkey = create_string_buffer("dkey {0}".format(dkey)) c_akey = create_string_buffer("akey {0}".format(akey)) val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata)+1) if indata != (repr(val.value)[1:-1]): self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail("ERROR: Data mismatch for dkey = {0}, akey={1}" .format("dkey {0}".format(dkey), "akey {0}".format(akey))) self.obj.close() self.container.close() def prepare_cont_ior_write_read(self, oclass, flags): """This method prepares the containers for IOR write and read invocations. To enable aggregation: - Create two containers and read always from first container Normal usage (use only a single container): - Create a single container and use the same. Args: oclass (str): IOR object class flags (str): IOR flags """ self.log.info(self.pool_cont_dict) # If pool is not in the dictionary, # initialize its container list to None # {poolA : [None, None], [None, None]} if self.pool not in self.pool_cont_dict: self.pool_cont_dict[self.pool] = [None] * 4 # Create container if the pool doesn't have one. # Otherwise, use the existing container in the pool. # pool_cont_dict {pool A: [containerA, Updated, # containerB, Updated], # pool B : containerA, Updated, # containerB, None]} if self.pool_cont_dict[self.pool][0] is None: self.add_container(self.pool, create=False) self.set_cont_class_properties(oclass) if self.test_with_checksum is False: tmp = self.get_object_replica_value(oclass) rf_value = "rf:{}".format(tmp - 1) self.update_cont_properties(rf_value) self.container.create() self.pool_cont_dict[self.pool][0] = self.container self.pool_cont_dict[self.pool][1] = "Updated" else: if ((self.test_during_aggregation is True) and (self.pool_cont_dict[self.pool][1] == "Updated") and (self.pool_cont_dict[self.pool][3] is None) and ("-w" in flags)): # Write to the second container self.add_container(self.pool, create=False) self.set_cont_class_properties(oclass) if self.test_with_checksum is False: tmp = self.get_object_replica_value(oclass) rf_value = "rf:{}".format(tmp - 1) self.update_cont_properties(rf_value) self.container.create() self.pool_cont_dict[self.pool][2] = self.container self.pool_cont_dict[self.pool][3] = "Updated" else: self.container = self.pool_cont_dict[self.pool][0] def delete_extra_container(self, pool): """Delete the extra container in the pool. Refer prepare_cont_ior_write_read. This method should be called when OSA tests intend to enable aggregation. Args: pool (object): pool handle """ self.pool.set_property("reclaim", "time") extra_container = self.pool_cont_dict[pool][2] extra_container.destroy() self.pool_cont_dict[pool][3] = None def get_object_replica_value(self, oclass): """ Get the object replica value for an object class. Args: oclass (str): Object Class (eg: RP_2G1,etc) Returns: value (int) : Object replica value """ value = 0 if "_" in oclass: replica_list = oclass.split("_") value = replica_list[1][0] else: self.log.info("Wrong Object Class. Cannot split") return int(value) def update_cont_properties(self, cont_prop): """Update the existing container properties. Args: cont_prop (str): Replace existing container properties with new value """ self.container.properties.value = cont_prop def set_cont_class_properties(self, oclass="S1"): """Update the container class to match the IOR/Mdtest object class. Fix the rf factor based on object replica value. Also, remove the redundancy factor for S type object class. Args: oclass (str, optional): Container object class to be set. Defaults to "S1". """ self.container.oclass.value = oclass # Set the container properties properly for S!, S2 class. # rf should not be set to 1 for S type object class. x = re.search("^S\\d$", oclass) prop = self.container.properties.value if x is not None: prop = prop.replace("rf:1", "rf:0") else: tmp = self.get_object_replica_value(oclass) rf_value = "rf:{}".format(tmp - 1) prop = prop.replace("rf:1", rf_value) self.container.properties.value = prop # Over-write oclass settings if using redundancy factor # and self.test_with_rf is True. # This has to be done so that container created doesn't # use the object class. if self.test_with_rf is True and \ "rf" in self.container.properties.value: self.log.info( "Detected container redundancy factor: %s", self.container.properties.value) self.ior_cmd.dfs_oclass.update(None, "ior.dfs_oclass") self.ior_cmd.dfs_dir_oclass.update(None, "ior.dfs_dir_oclass") self.container.oclass.update(None) def assert_on_exception(self, out_queue=None): """Assert on exception while executing an application. Args: out_queue (queue): Check whether the queue is empty. If empty, app (ior, mdtest) didn't encounter error. """ if out_queue is None: out_queue = self.out_queue if out_queue.empty(): pass else: exc = out_queue.get(block=False) out_queue.put(exc) raise CommandFailure(exc) def cleanup_queue(self, out_queue=None): """Cleanup the existing thread queue. Args: out_queue (queue): Queue to cleanup. """ if out_queue is None: out_queue = self.out_queue while not out_queue.empty(): out_queue.get(block=True) def run_ior_thread(self, action, oclass, test, single_cont_read=True, fail_on_warning=True, pool=None): """Start the IOR thread for either writing or reading data to/from a container. Args: action (str): Start the IOR thread with Read or Write oclass (str): IOR object class test (list): IOR test sequence flags (str): IOR flags single_cont_read (bool) : Always read from the 1st container. Defaults to True. fail_on_warning (bool) : Test terminates for IOR warnings. Defaults to True. pool (TestPool): Pool to run ior on. Defaults to None. """ # Intermediate (between correct and hack) implementation for allowing a # pool to be passed in. Needs to be fixed by making the pool argument # required. if pool is None: pool = self.pool self.cleanup_queue() if action == "Write": flags = self.ior_w_flags else: flags = self.ior_r_flags # Add a thread for these IOR arguments process = threading.Thread(target=self.ior_thread, kwargs={"pool": pool, "oclass": oclass, "test": test, "flags": flags, "single_cont_read": single_cont_read, "fail_on_warning": fail_on_warning}) # Launch the IOR thread process.start() # Wait for the thread to finish process.join() if not self.out_queue.empty(): self.assert_on_exception() def ior_thread(self, pool, oclass, test, flags, single_cont_read=True, fail_on_warning=True): """Start an IOR thread. Args: pool (object): pool handle oclass (str): IOR object class, container class. test (list): IOR test sequence flags (str): IOR flags single_cont_read (bool) : Always read from the 1st container. Defaults to True. fail_on_warning (bool) : Test terminates for IOR warnings. Defaults to True. """ self.cleanup_queue() self.pool = pool self.ior_cmd.get_params(self) self.ior_cmd.set_daos_params(self.server_group, self.pool) self.log.info("Redundancy Factor : %s", self.test_with_rf) self.ior_cmd.dfs_oclass.update(oclass) self.ior_cmd.dfs_dir_oclass.update(oclass) if single_cont_read is True: # Prepare the containers created and use in a specific # way defined in prepare_cont_ior_write. self.prepare_cont_ior_write_read(oclass, flags) elif single_cont_read is False and self.container is not None: # Here self.container is having actual value. Just use it. self.log.info(self.container) else: self.fail("Not supported option on ior_thread") try: job_manager = self.get_ior_job_manager_command() except CommandFailure as err_msg: self.out_queue.put(err_msg) self.assert_on_exception() job_manager.job.dfs_cont.update(self.container.uuid) self.ior_cmd.transfer_size.update(test[2]) self.ior_cmd.block_size.update(test[3]) self.ior_cmd.flags.update(flags) # Update oclass settings if using redundancy factor # and self.test_with_rf is True. if self.test_with_rf is True and \ "rf" in self.container.properties.value: self.log.info( "Detected container redundancy factor: %s", self.container.properties.value) self.ior_cmd.dfs_oclass.update(None, "ior.dfs_oclass") self.ior_cmd.dfs_dir_oclass.update(None, "ior.dfs_dir_oclass") self.run_ior_with_pool(create_pool=False, create_cont=False, fail_on_warning=fail_on_warning, out_queue=self.out_queue) if not self.out_queue.empty(): self.assert_on_exception() def run_mdtest_thread(self, oclass="RP_2G1"): """Start mdtest thread and wait until thread completes. Args: oclass (str): IOR object class, container class. """ # Create container only self.mdtest_cmd.dfs_destroy = False create_container = 0 if self.container is None: self.add_container(self.pool, create=False) create_container = 1 self.mdtest_cmd.dfs_oclass.update(oclass) self.set_cont_class_properties(oclass) if self.test_with_checksum is False: tmp = self.get_object_replica_value(oclass) rf_value = "rf:{}".format(tmp - 1) self.update_cont_properties(rf_value) if create_container == 1: self.container.create() job_manager = self.get_mdtest_job_manager_command(self.manager) job_manager.job.dfs_cont.update(self.container.uuid) # Add a thread for these IOR arguments process = threading.Thread(target=self.execute_mdtest) # Launch the MDtest thread process.start() # Wait for the thread to finish process.join() if not self.out_queue.empty(): self.assert_on_exception()
def test_array_obj(self): """ Test ID: DAOS-961 Test Description: Writes an array to an object and then reads it back and verifies it. :avocado: tags=all,smoke,pr,object,tiny,basicobject """ try: # parameters used in pool create createmode = self.params.get("mode", '/run/pool_params/createmode/') createsetid = self.params.get("setname", '/run/pool_params/createset/') createsize = self.params.get("size", '/run/pool_params/createsize/') createuid = os.geteuid() creategid = os.getegid() # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) self.plog.info("Pool %s created.", pool.get_uuid_str()) # need a connection to create container pool.connect(1 << 1) # create a container container = DaosContainer(self.context) container.create(pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # do a query and compare the UUID returned from create with # that returned by query container.query() if container.get_uuid_str() != c_uuid_to_str( container.info.ci_uuid): self.fail("Container UUID did not match the one in info\n") # create an object and write some data into it thedata = [] thedata.append("data string one") thedata.append("data string two") thedata.append("data string tre") dkey = "this is the dkey" akey = "this is the akey" self.plog.info("writing array to dkey >%s< akey >%s<.", dkey, akey) oid, epoch = container.write_an_array_value(thedata, dkey, akey, obj_cls=3) # read the data back and make sure its correct length = len(thedata[0]) thedata2 = container.read_an_array(len(thedata), length + 1, dkey, akey, oid, epoch) if thedata[0][0:length - 1] != thedata2[0][0:length - 1]: self.plog.error("Data mismatch") self.plog.error("Wrote: >%s<", thedata[0]) self.plog.error("Read: >%s<", thedata2[0]) self.fail("Write data, read it back, didn't match\n") if thedata[2][0:length - 1] != thedata2[2][0:length - 1]: self.plog.error("Data mismatch") self.plog.error("Wrote: >%s<", thedata[2]) self.plog.error("Read: >%s<", thedata2[2]) self.fail("Write data, read it back, didn't match\n") container.close() # wait a few seconds and then destroy time.sleep(5) container.destroy() # cleanup the pool pool.disconnect() pool.destroy(1) self.plog.info("Test Complete") except DaosApiError as excep: self.plog.error("Test Failed, exception was thrown.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to pass but it failed.\n")
def test_null_values(self): """ Test ID: DAOS-1376 Test Description: Pass a dkey and an akey that is null. :avocado: tags=all,object,full_regression,small,objupdatenull """ self.prepare_pool() try: # create a container container = DaosContainer(self.context) container.create(self.pool.pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # data used in the test thedata = "a string that I want to stuff into an object" thedatasize = len(thedata) + 1 except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test failed during setup .\n") try: # try using a null dkey dkey = None akey = "this is the akey" container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) container.close() container.destroy() self.plog.error("Didn't get expected return code.") self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): container.close() container.destroy() self.plog.error("Didn't get expected return code.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # try using a null akey/io descriptor dkey = "this is the dkey" akey = None container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): self.plog.error("Didn't get expected return code.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # lastly try passing no data thedata = None thedatasize = 0 dkey = "this is the dkey" akey = "this is the akey" container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.plog.info("Update with no data worked") except DaosApiError as excep: container.close() container.destroy() print(excep) print(traceback.format_exc()) self.plog.error("Update with no data failed") self.fail("Update with no data failed.\n") container.close() container.destroy() self.plog.info("Test Complete")
def test_tx_basics(self): """ Perform I/O to an object in a container in 2 different transactions, verifying basic I/O and transactions in particular. NOTE: this was an epoch test and all I did was get it working with tx Not a good test at this point, need to redesign when tx is fully working. :avocado: tags=all,container,tx,small,smoke,pr,basictx """ self.pool = None try: # parameters used in pool create createmode = self.params.get("mode", '/run/poolparams/createmode/') createuid = os.geteuid() creategid = os.getegid() createsetid = self.params.get("setname", '/run/poolparams/createset/') createsize = self.params.get("size", '/run/poolparams/createsize/') # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) # need a connection to create container self.pool.connect(1 << 1) # create a container container = DaosContainer(self.context) container.create(self.pool.handle) # now open it container.open() # do a query and compare the UUID returned from create with # that returned by query container.query() if container.get_uuid_str() != c_uuid_to_str( container.info.ci_uuid): self.fail("Container UUID did not match the one in info\n") # create an object and write some data into it thedata = "a string that I want to stuff into an object" thedatasize = 45 dkey = "this is the dkey" akey = "this is the akey" oid, txn = container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) # read the data back and make sure its correct thedata2 = container.read_an_obj(thedatasize, dkey, akey, oid, txn) if thedata != thedata2.value: print("thedata>" + thedata) print("thedata2>" + thedata2.value) self.fail("Write data 1, read it back, didn't match\n") # repeat above, but know that the write_an_obj call is advancing # the epoch so the original copy remains and the new copy is in # a new epoch. thedata3 = "a different string" thedatasize2 = 19 # note using the same keys so writing to the same spot dkey = "this is the dkey" akey = "this is the akey" oid, tx2 = container.write_an_obj(thedata3, thedatasize2, dkey, akey, oid, None, 2) # read the data back and make sure its correct thedata4 = container.read_an_obj(thedatasize2, dkey, akey, oid, tx2) if thedata3 != thedata4.value: self.fail("Write data 2, read it back, didn't match\n") # transactions generally don't work this way but need to explore # an alternative to below code once model is complete, maybe # read from a snapshot or read from TX_NONE etc. # the original data should still be there too #thedata5 = container.read_an_obj(thedatasize, dkey, akey, # oid, transaction) #if thedata != thedata5.value: # self.fail("Write data 3, read it back, didn't match\n") container.close() # wait a few seconds and then destroy time.sleep(5) container.destroy() except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test was expected to pass but it failed.\n")
class CreateManyDkeys(Test): """ Test Class Description: Tests that create large numbers of keys in objects/containers and then destroy the containers and verify the space has been reclaimed. :avocado: recursive """ def setUp(self): super(CreateManyDkeys, self).setUp() self.pool = DaosPool(self.context) self.pool.create(self.params.get("mode", '/run/pool/createmode/*'), os.geteuid(), os.getegid(), self.params.get("size", '/run/pool/createsize/*'), self.params.get("setname", '/run/pool/createset/*'), None) self.pool.connect(1 << 1) def tearDown(self): try: if self.pool: self.pool.disconnect() self.pool.destroy(1) finally: super(CreateManyDkeys, self).tearDown() def write_a_bunch_of_values(self, how_many): """ Write data to an object, each with a dkey and akey. The how_many parameter determines how many key:value pairs are written. """ self.container = DaosContainer(self.context) self.container.create(self.pool.handle) self.container.open() ioreq = IORequest(self.context, self.container, None) epoch = self.container.get_new_tx() c_epoch = ctypes.c_uint64(epoch) print("Started Writing the Dataset-----------\n") inc = 50000 last_key = inc for key in range(how_many): c_dkey = ctypes.create_string_buffer("dkey {0}".format(key)) c_akey = ctypes.create_string_buffer("akey {0}".format(key)) c_value = ctypes.create_string_buffer( "some data that gets stored with the key {0}".format(key)) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) ioreq.single_insert(c_dkey, c_akey, c_value, c_size, c_epoch) if key > last_key: print("written: {}".format(key)) sys.stdout.flush() last_key = key + inc self.container.commit_tx(c_epoch) print("Started Verification of the Dataset-----------\n") last_key = inc for key in range(how_many): c_dkey = ctypes.create_string_buffer("dkey {0}".format(key)) c_akey = ctypes.create_string_buffer("akey {0}".format(key)) the_data = "some data that gets stored with the key {0}".format( key) val = ioreq.single_fetch(c_dkey, c_akey, len(the_data) + 1, c_epoch) if the_data != (repr(val.value)[1:-1]): self.fail("ERROR: Data mismatch for dkey = {0}, akey={1}, " "Expected Value={2} and Received Value={3}\n".format( "dkey {0}".format(key), "akey {0}".format(key), the_data, repr(val.value)[1:-1])) if key > last_key: print("veried: {}".format(key)) sys.stdout.flush() last_key = key + inc print("starting destroy") self.container.close() self.container.destroy() print("destroy complete") @avocado.fail_on(DaosApiError) @skipForTicket("DAOS-1721") def test_many_dkeys(self): """ Test ID: DAOS-1701 Test Description: Test many of dkeys in same object. Use Cases: 1. large key counts 2. space reclaimation after destroy :avocado: tags=all,full,small,object,many_dkeys """ no_of_dkeys = self.params.get("number_of_dkeys", '/run/dkeys/') # write a lot of individual data items, verify them, then destroy self.write_a_bunch_of_values(no_of_dkeys) # do it again, which should verify the first container # was truely destroyed because a second round won't fit otherwise self.write_a_bunch_of_values(no_of_dkeys)
def test_null_values(self): """ Test ID: DAOS-1376 Test Description: Pass a dkey and an akey that is null. :avocado: tags=all,object,full_regression,small,objupdatenull """ try: # parameters used in pool create createmode = self.params.get("mode", '/run/conttests/createmode/') createsetid = self.params.get("setname", '/run/conttests/createset/') createsize = self.params.get("size", '/run/conttests/createsize/') createuid = os.geteuid() creategid = os.getegid() # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) self.plog.info("Pool %s created.", pool.get_uuid_str()) # need a connection to create container pool.connect(1 << 1) # create a container container = DaosContainer(self.context) container.create(pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # data used in the test thedata = "a string that I want to stuff into an object" thedatasize = len(thedata) + 1 except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test failed during setup .\n") try: # try using a null dkey dkey = None akey = "this is the akey" container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) container.close() container.destroy() pool.disconnect() pool.destroy(1) self.plog.error("Didn't get expected return code.") self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): container.close() container.destroy() pool.disconnect() pool.destroy(1) self.plog.error("Didn't get expected return code.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # try using a null akey/io descriptor dkey = "this is the dkey" akey = None container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): self.plog.error("Didn't get expected return code.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # lastly try passing no data thedata = None thedatasize = 0 dkey = "this is the dkey" akey = "this is the akey" container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.plog.info("Update with no data worked") except DaosApiError as excep: container.close() container.destroy() pool.disconnect() pool.destroy(1) print(excep) print(traceback.format_exc()) self.plog.error("Update with no data failed") self.fail("Update with no data failed.\n") container.close() container.destroy() pool.disconnect() pool.destroy(1) self.plog.info("Test Complete")
class CreateManyDkeys(TestWithServers): """ Test Class Description: Tests that create large numbers of keys in objects/containers and then destroy the containers and verify the space has been reclaimed. :avocado: recursive """ def write_a_bunch_of_values(self, how_many): """ Write data to an object, each with a dkey and akey. The how_many parameter determines how many key:value pairs are written. """ self.container = DaosContainer(self.context) self.container.create(self.pool.pool.handle) self.container.open() ioreq = IORequest(self.context, self.container, None) print("Started Writing the Dataset-----------\n") inc = 50000 last_key = inc for key in range(how_many): c_dkey = create_string_buffer("dkey {0}".format(key)) c_akey = create_string_buffer("akey {0}".format(key)) c_value = create_string_buffer( "some data that gets stored with the key {0}".format(key)) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) ioreq.single_insert(c_dkey, c_akey, c_value, c_size) if key > last_key: print("written: {}".format(key)) sys.stdout.flush() last_key = key + inc print("Started Verification of the Dataset-----------\n") last_key = inc for key in range(how_many): c_dkey = create_string_buffer("dkey {0}".format(key)) c_akey = create_string_buffer("akey {0}".format(key)) the_data = "some data that gets stored with the key {0}".format(key) val = ioreq.single_fetch(c_dkey, c_akey, len(the_data)+1) exp_value = val.value.decode("utf-8") if the_data != exp_value: self.fail("ERROR: Data mismatch for dkey = {0}, akey={1}, " "Expected Value={2} and Received Value={3}\n" .format("dkey {0}".format(key), "akey {0}".format(key), the_data, exp_value)) if key > last_key: print("veried: {}".format(key)) sys.stdout.flush() last_key = key + inc print("starting destroy") self.container.close() self.container.destroy() print("destroy complete") @avocado.fail_on(DaosApiError) def test_many_dkeys(self): """ Test ID: DAOS-1701 Test Description: Test many of dkeys in same object. Use Cases: 1. large key counts 2. space reclamation after destroy :avocado: tags=all,full_regression :avocado: tags=small :avocado: tags=object :avocado: tags=many_dkeys """ self.prepare_pool() no_of_dkeys = self.params.get("number_of_dkeys", '/run/dkeys/') # write a lot of individual data items, verify them, then destroy self.write_a_bunch_of_values(no_of_dkeys) # do it again, which should verify the first container # was truly destroyed because a second round won't fit otherwise self.write_a_bunch_of_values(no_of_dkeys)
class OSAUtils(MdtestBase, IorTestBase): # pylint: disable=too-many-ancestors """ Test Class Description: This test runs daos_server offline drain test cases. :avocado: recursive """ def setUp(self): """Set up for test case.""" super().setUp() self.pool_cont_dict = {} self.container = None self.obj = None self.ioreq = None self.dmg_command = self.get_dmg_command() self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*', default=[0])[0] self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*', default=[0])[0] self.record_length = self.params.get("length", '/run/record/*', default=[0])[0] self.ior_w_flags = self.params.get("write_flags", '/run/ior/iorflags/*', default="") self.ior_r_flags = self.params.get("read_flags", '/run/ior/iorflags/*') self.out_queue = queue.Queue() self.dmg_command.exit_status_exception = False self.test_during_aggregation = False self.test_during_rebuild = False self.test_with_checksum = True @fail_on(CommandFailure) def get_pool_leader(self): """Get the pool leader. Returns: int: pool leader value """ data = self.dmg_command.pool_query(self.pool.uuid) return int(data["response"]["leader"]) @fail_on(CommandFailure) def get_rebuild_status(self): """Get the rebuild status. Returns: str: reuild status """ data = self.dmg_command.pool_query(self.pool.uuid) return data["response"]["rebuild"]["status"] @fail_on(CommandFailure) def is_rebuild_done(self, time_interval, wait_for_rebuild_to_complete=False): """Rebuild is completed/done. Args: time_interval: Wait interval between checks wait_for_rebuild_to_complete: Rebuild completed (Default: False) """ self.pool.wait_for_rebuild(wait_for_rebuild_to_complete, interval=time_interval) @fail_on(CommandFailure) def assert_on_rebuild_failure(self): """If the rebuild is not successful, raise assert. """ rebuild_status = self.get_rebuild_status() self.log.info("Rebuild Status: %s", rebuild_status) rebuild_failed_string = ["failed", "scanning", "aborted", "busy"] self.assertTrue(rebuild_status not in rebuild_failed_string, "Rebuild failed") @fail_on(CommandFailure) def print_and_assert_on_rebuild_failure(self, out, timeout=3): """Print the out value (daos, dmg, etc) and check for rebuild completion. If not, raise assert. """ self.log.info(out) self.is_rebuild_done(timeout) self.assert_on_rebuild_failure() @fail_on(CommandFailure) def get_pool_version(self): """Get the pool version. Returns: int: pool_version_value """ data = self.dmg_command.pool_query(self.pool.uuid) return int(data["response"]["version"]) def set_container(self, container): """Set the OSA utils container object. Args: container (obj) : Container object to be used within OSA utils. """ self.container = container def simple_exclude_reintegrate_loop(self, rank, loop_time=100): """This method performs exclude and reintegration on a rank, for a certain amount of time. """ start_time = 0 finish_time = 0 while int(finish_time - start_time) > loop_time: start_time = time.time() output = self.dmg_command.pool_exclude(self.pool.uuid, rank) self.print_and_assert_on_rebuild_failure(output) output = self.dmg_command.pool_reintegrate(self.pool.uuid, rank) self.print_and_assert_on_rebuild_failure(output) @fail_on(DaosApiError) def write_single_object(self): """Write some data to the existing pool.""" self.pool.connect(2) csum = self.params.get("enable_checksum", '/run/container/*') self.container = DaosContainer(self.context) input_param = self.container.cont_input_values input_param.enable_chksum = csum self.container.create(poh=self.pool.pool.handle, con_prop=input_param) self.container.open() self.obj = DaosObj(self.context, self.container) self.obj.create(objcls=1) self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) self.log.info("Writing the Single Dataset") for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length) d_key_value = "dkey {0}".format(dkey) c_dkey = create_string_buffer(d_key_value) a_key_value = "akey {0}".format(akey) c_akey = create_string_buffer(a_key_value) c_value = create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size) self.obj.close() self.container.close() @fail_on(DaosApiError) def verify_single_object(self): """Verify the container data on the existing pool.""" self.pool.connect(2) self.container.open() self.obj.open() self.log.info("Single Dataset Verification -- Started") for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length) c_dkey = create_string_buffer("dkey {0}".format(dkey)) c_akey = create_string_buffer("akey {0}".format(akey)) val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1) if indata != (repr(val.value)[1:-1]): self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR: Data mismatch for dkey = {0}, akey={1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.obj.close() self.container.close() def prepare_cont_ior_write_read(self, oclass, flags): """This method prepares the containers for IOR write and read invocations. To enable aggregation: - Create two containers and read always from first container Normal usage (use only a single container): - Create a single container and use the same. Args: oclass (str): IOR object class flags (str): IOR flags """ self.log.info(self.pool_cont_dict) # If pool is not in the dictionary, # initialize its container list to None # {poolA : [None, None], [None, None]} if self.pool not in self.pool_cont_dict: self.pool_cont_dict[self.pool] = [None] * 4 # Create container if the pool doesn't have one. # Otherwise, use the existing container in the pool. # pool_cont_dict {pool A: [containerA, Updated, # containerB, Updated], # pool B : containerA, Updated, # containerB, None]} if self.pool_cont_dict[self.pool][0] is None: self.add_container(self.pool, create=False) self.set_cont_class_properties(oclass) if self.test_with_checksum is False: tmp = self.get_object_replica_value(oclass) rf_value = "rf:{}".format(tmp - 1) self.update_cont_properties(rf_value) self.container.create() self.pool_cont_dict[self.pool][0] = self.container self.pool_cont_dict[self.pool][1] = "Updated" else: if ((self.test_during_aggregation is True) and (self.pool_cont_dict[self.pool][1] == "Updated") and (self.pool_cont_dict[self.pool][3] is None) and ("-w" in flags)): # Write to the second container self.add_container(self.pool, create=False) self.set_cont_class_properties(oclass) if self.test_with_checksum is False: tmp = self.get_object_replica_value(oclass) rf_value = "rf:{}".format(tmp - 1) self.update_cont_properties(rf_value) self.container.create() self.pool_cont_dict[self.pool][2] = self.container self.pool_cont_dict[self.pool][3] = "Updated" else: self.container = self.pool_cont_dict[self.pool][0] def delete_extra_container(self, pool): """Delete the extra container in the pool. Refer prepare_cont_ior_write_read. This method should be called when OSA tests intend to enable aggregation. Args: pool (object): pool handle """ self.pool.set_property("reclaim", "time") extra_container = self.pool_cont_dict[pool][2] extra_container.destroy() self.pool_cont_dict[pool][3] = None def get_object_replica_value(self, oclass): """ Get the object replica value for an object class. Args: oclass (str): Object Class (eg: RP_2G1,etc) Returns: value (int) : Object replica value """ value = 0 if "_" in oclass: replica_list = oclass.split("_") value = replica_list[1][0] else: self.log.info("Wrong Object Class. Cannot split") return int(value) def update_cont_properties(self, cont_prop): """Update the existing container properties. Args: cont_prop (str): Replace existing container properties with new value """ self.container.properties.value = cont_prop def set_cont_class_properties(self, oclass="S1"): """Update the container class to match the IOR object class. Fix the rf factor based on object replica value. Also, remove the redundancy factor for S type object class. Args: oclass (str, optional): Container object class to be set. Defaults to "S1". """ self.container.oclass.value = oclass # Set the container properties properly for S!, S2 class. # rf should not be set to 1 for S type object class. x = re.search("^S\\d$", oclass) prop = self.container.properties.value if x is not None: prop = prop.replace("rf:1", "rf:0") else: tmp = self.get_object_replica_value(oclass) rf_value = "rf:{}".format(tmp - 1) prop = prop.replace("rf:1", rf_value) self.container.properties.value = prop def assert_on_exception(self, out_queue=None): """Assert on exception while executing an application. Args: out_queue (queue): Check whether the queue is empty. If empty, app (ior, mdtest) didn't encounter error. """ if out_queue is None: out_queue = self.out_queue if out_queue.empty(): pass else: exc = out_queue.get(block=False) out_queue.put(exc) raise exc def cleanup_queue(self, out_queue=None): """Cleanup the existing thread queue. Args: out_queue (queue): Queue to cleanup. """ if out_queue is None: out_queue = self.out_queue while not out_queue.empty(): out_queue.get(block=True) def run_ior_thread(self, action, oclass, test, single_cont_read=True, fail_on_warning=True): """Start the IOR thread for either writing or reading data to/from a container. Args: action (str): Start the IOR thread with Read or Write oclass (str): IOR object class test (list): IOR test sequence flags (str): IOR flags single_cont_read (bool) : Always read from the 1st container. Defaults to True. fail_on_warning (bool) : Test terminates for IOR warnings. Defaults to True. """ self.cleanup_queue() if action == "Write": flags = self.ior_w_flags else: flags = self.ior_r_flags # Add a thread for these IOR arguments process = threading.Thread(target=self.ior_thread, kwargs={ "pool": self.pool, "oclass": oclass, "test": test, "flags": flags, "single_cont_read": single_cont_read, "fail_on_warning": fail_on_warning }) # Launch the IOR thread process.start() # Wait for the thread to finish try: process.join() except CommandFailure as err_msg: self.out_queue.put(err_msg) self.assert_on_exception() def ior_thread(self, pool, oclass, test, flags, single_cont_read=True, fail_on_warning=True): """Start an IOR thread. Args: pool (object): pool handle oclass (str): IOR object class, container class. test (list): IOR test sequence flags (str): IOR flags single_cont_read (bool) : Always read from the 1st container. Defaults to True. fail_on_warning (bool) : Test terminates for IOR warnings. Defaults to True. """ self.cleanup_queue() self.pool = pool self.ior_cmd.get_params(self) self.ior_cmd.set_daos_params(self.server_group, self.pool) self.ior_cmd.dfs_oclass.update(oclass) self.ior_cmd.dfs_dir_oclass.update(oclass) if single_cont_read is True: # Prepare the containers created and use in a specific # way defined in prepare_cont_ior_write. self.prepare_cont_ior_write_read(oclass, flags) elif single_cont_read is False and self.container is not None: # Here self.container is having actual value. Just use it. self.log.info(self.container) else: self.fail("Not supported option on ior_thread") try: job_manager = self.get_ior_job_manager_command() except CommandFailure as err_msg: self.out_queue.put(err_msg) self.assert_on_exception() job_manager.job.dfs_cont.update(self.container.uuid) self.ior_cmd.transfer_size.update(test[2]) self.ior_cmd.block_size.update(test[3]) self.ior_cmd.flags.update(flags) try: self.run_ior_with_pool(create_pool=False, create_cont=False, fail_on_warning=fail_on_warning) except CommandFailure as err_msg: self.out_queue.put(err_msg) self.assert_on_exception() def run_mdtest_thread(self): """Start mdtest thread and wait until thread completes. """ # Create container only self.mdtest_cmd.dfs_destroy = False if self.container is None: self.add_container(self.pool, create=False) self.set_cont_class_properties(self.mdtest_cmd.dfs_oclass) if self.test_with_checksum is False: tmp = self.get_object_replica_value(self.mdtest_cmd.dfs_oclass) rf_value = "rf:{}".format(tmp - 1) self.update_cont_properties(rf_value) self.container.create() job_manager = self.get_mdtest_job_manager_command(self.manager) job_manager.job.dfs_cont.update(self.container.uuid) # Add a thread for these IOR arguments process = threading.Thread(target=self.execute_mdtest) # Launch the MDtest thread process.start() # Wait for the thread to finish try: process.join() except CommandFailure as err_msg: self.out_queue.put(err_msg) self.assert_on_exception()
class ContainerAsync(TestWithServers): """ Tests DAOS pool connect permissions (non existing pool handle, bad uuid) and close. :avocado: recursive """ def __init__(self, *args, **kwargs): super(ContainerAsync, self).__init__(*args, **kwargs) self.container1 = None self.container2 = None self.pool = None def test_createasync(self): """ Test container create for asynchronous mode. :avocado: tags=all,small,full_regression,container,createasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) GLOB_SIGNAL = threading.Event() self.container1.create(poh, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test") print("RC after successful container create: ", GLOB_RC) # Try to recreate container after destroying pool, # this should fail. Checking rc after failure. self.pool.disconnect() self.pool.destroy(1) GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.create(poh, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC == 0: self.fail("RC not as expected in async test") print("RC after unsuccessful container create: ", GLOB_RC) # cleanup the pool and container self.pool = None except DaosApiError as excep: print(excep) print(traceback.format_exc()) def test_destroyasync(self): """ Test container destroy for asynchronous mode. :avocado: tags=all,small,full_regression,container,contdestroyasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) self.container1.create(poh) GLOB_SIGNAL = threading.Event() self.container1.destroy(1, poh, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test") print("RC after successful container create: ", GLOB_RC) # Try to destroy container again, this should fail, as non-existent. # Checking rc after failure. GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.destroy(1, poh, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != -1003: self.fail("RC not as expected in async test") print("RC after container destroy failed:", GLOB_RC) except DaosApiError as excep: print(excep) print(traceback.format_exc()) def test_openasync(self): """ Test container open for asynchronous mode. :avocado: tags=all,small,full_regression,container,openasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) self.container1.create(poh) str_cuuid = self.container1.get_uuid_str() cuuid = uuid.UUID(str_cuuid) GLOB_SIGNAL = threading.Event() self.container1.open(poh, cuuid, 2, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test") print("RC after successful container create: ", GLOB_RC) # Try to open container2, this should fail, as non-existent. # Checking rc after failure. GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.open(None, None, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC == 0: self.fail("RC not as expected in async test") print("RC after container destroy failed:", GLOB_RC) # cleanup the container self.container1.close() self.container1.destroy() except DaosApiError as excep: print(excep) print(traceback.format_exc()) def test_closeasync(self): """ Test container close for asynchronous mode. :avocado: tags=all,small,full_regression,container,closeasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) self.container1.create(poh) str_cuuid = self.container1.get_uuid_str() cuuid = uuid.UUID(str_cuuid) self.container1.open(poh, cuuid, 2) GLOB_SIGNAL = threading.Event() self.container1.close(cb_func=cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test: " "{0}".format(GLOB_RC)) print("RC after successful container create: ", GLOB_RC) # Try to open container2, this should fail, as non-existent. # Checking rc after failure. GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.close(cb_func=cb_func) GLOB_SIGNAL.wait() if GLOB_RC == 0: self.fail("RC not as expected in async test: " "{0}".format(GLOB_RC)) print("RC after container destroy failed:", GLOB_RC) # cleanup the container self.container1.destroy() except DaosApiError as excep: print(excep) print(traceback.format_exc()) def test_queryasync(self): """ Test container query for asynchronous mode. :avocado: tags=all,small,full_regression,container,queryasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) self.container1.create(poh) dummy_str_cuuid = self.container1.get_uuid_str() # Open container self.container1.open(poh, None, 2, None) GLOB_SIGNAL = threading.Event() self.container1.query(cb_func=cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test: " "{0}".format(GLOB_RC)) print("RC after successful container create: ", GLOB_RC) # Close opened container self.container1.close() # Try to open container2, this should fail, as non-existent. # Checking rc after failure. GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.query(cb_func=cb_func) GLOB_SIGNAL.wait() if GLOB_RC == 0: self.fail("RC not as expected in async test: " "{0}".format(GLOB_RC)) print("RC after container destroy failed:", GLOB_RC) # cleanup the container self.container1.destroy() except DaosApiError as excep: print(excep) print(traceback.format_exc())
class FullPoolContainerCreate(TestWithServers): """ Class for test to create a container in a pool with no remaining free space. :avocado: recursive """ def __init__(self, *args, **kwargs): super(FullPoolContainerCreate, self).__init__(*args, **kwargs) self.cont = None self.cont2 = None @skipForTicket("DAOS-3142") def test_no_space_cont_create(self): """ :avocado: tags=all,container,tiny,full_regression,fullpoolcontcreate """ # full storage rc err = "-1007" # probably should be -1007, revisit later err2 = "-1009" # create pool self.pool = DaosPool(self.context) mode = self.params.get("mode", '/conttests/createmode/') self.d_log.debug("mode is {0}".format(mode)) uid = os.geteuid() gid = os.getegid() # 16 mb pool, minimum size currently possible size = 16777216 self.d_log.debug("creating pool") self.pool.create(mode, uid, gid, size, self.server_group, None) self.d_log.debug("created pool") # connect to the pool self.d_log.debug("connecting to pool") self.pool.connect(1 << 1) self.d_log.debug("connected to pool") # query the pool self.d_log.debug("querying pool info") dummy_pool_info = self.pool.pool_query() self.d_log.debug("queried pool info") # create a container try: self.d_log.debug("creating container") self.cont = DaosContainer(self.context) self.cont.create(self.pool.handle) self.d_log.debug("created container") except DaosApiError as excep: self.d_log.error("caught exception creating container: " "{0}".format(excep)) self.fail("caught exception creating container: {0}".format(excep)) self.d_log.debug("opening container") self.cont.open() self.d_log.debug("opened container") # generate random dkey, akey each time # write 1mb until no space, then 1kb, etc. to fill pool quickly for obj_sz in [1048576, 10240, 10, 1]: write_count = 0 while True: self.d_log.debug("writing obj {0}, sz {1} to " "container".format(write_count, obj_sz)) my_str = "a" * obj_sz my_str_sz = obj_sz dkey = (''.join( random.choice(string.lowercase) for i in range(5))) akey = (''.join( random.choice(string.lowercase) for i in range(5))) try: dummy_oid = self.cont.write_an_obj(my_str, my_str_sz, dkey, akey, obj_cls="OC_SX") self.d_log.debug("wrote obj {0}, sz {1}".format( write_count, obj_sz)) write_count += 1 except DaosApiError as excep: if not (err in repr(excep) or err2 in repr(excep)): self.d_log.error("caught exception while writing " "object: {0}".format(repr(excep))) self.fail("caught exception while writing object: {0}". format(repr(excep))) else: self.d_log.debug("pool is too full for {0} byte " "objects".format(obj_sz)) break self.d_log.debug("closing container") self.cont.close() self.d_log.debug("closed container") # create a 2nd container now that pool is full try: self.d_log.debug("creating 2nd container") self.cont2 = DaosContainer(self.context) self.cont2.create(self.pool.handle) self.d_log.debug("created 2nd container") self.d_log.debug("opening container 2") self.cont2.open() self.d_log.debug("opened container 2") self.d_log.debug("writing one more object, write expected to fail") self.cont2.write_an_obj(my_str, my_str_sz, dkey, akey, obj_cls="OC_SX") self.fail("wrote one more object after pool was completely filled," " this should never print") except DaosApiError as excep: if not (err in repr(excep) or err2 in repr(excep)): self.d_log.error("caught unexpected exception while " "writing object: {0}".format(repr(excep))) self.fail("caught unexpected exception while writing " "object: {0}".format(repr(excep))) else: self.d_log.debug("correctly caught -1007 while attempting " "to write object in full pool")
class DeleteContainerTest(TestWithServers): """ Tests DAOS container delete and close. :avocado: recursive """ def test_container_delete(self): """ Test basic container delete :avocado: tags=all,container,tiny,smoke,full_regression,contdelete """ expected_for_param = [] uuidlist = self.params.get("uuid", '/run/createtests/ContainerUUIDS/*/') cont_uuid = uuidlist[0] expected_for_param.append(uuidlist[1]) pohlist = self.params.get("poh", '/run/createtests/PoolHandles/*/') poh = pohlist[0] expected_for_param.append(pohlist[1]) openlist = self.params.get("opened", "/run/createtests/ConnectionOpened/*/") opened = openlist[0] expected_for_param.append(openlist[1]) forcelist = self.params.get("force", "/run/createtests/ForceDestroy/*/") force = forcelist[0] # force=0 in .yaml file specifies FAIL, however: # if not opened and force=0 expect pass if force == 0 and not opened: expected_for_param.append('PASS') else: expected_for_param.append(forcelist[1]) # opened=True in .yaml file specifies PASS, however # if it is also the case force=0, then FAIL is expected expected_result = 'PASS' for result in expected_for_param: if result == 'FAIL': expected_result = 'FAIL' break # initialize a python pool object then create the underlying # daos storage and connect to it self.prepare_pool() passed = False try: self.container = DaosContainer(self.context) # create should always work (testing destroy) if not cont_uuid == 'INVALID': cont_uuid = uuid.UUID(uuidlist[0]) save_cont_uuid = cont_uuid self.container.create(self.pool.pool.handle, cont_uuid) else: self.container.create(self.pool.pool.handle) save_cont_uuid = uuid.UUID(self.container.get_uuid_str()) # Opens the container if required if opened: self.container.open(self.pool.pool.handle) # wait a few seconds and then attempts to destroy container time.sleep(5) if poh == 'VALID': poh = self.pool.pool.handle # if container is INVALID, overwrite with non existing UUID if cont_uuid == 'INVALID': cont_uuid = uuid.uuid4() self.container.destroy(force=force, poh=poh, con_uuid=cont_uuid) passed = True except DaosApiError as excep: self.log.info(excep, traceback.format_exc()) self.container.destroy(force=1, poh=self.pool.pool.handle, con_uuid=save_cont_uuid) finally: # close container handle, release a reference on pool in client lib # Otherwise test will ERROR in tearDown (pool disconnect -DER_BUSY) if opened: self.container.close() self.container = None if expected_result == 'PASS' and not passed: self.fail("Test was expected to pass but it failed.\n") if expected_result == 'FAIL' and passed: self.fail("Test was expected to fail but it passed.\n")
def test_no_space_cont_create(self): """ :avocado: tags=all,container,tiny,full_regression,fullpoolcontcreate """ # full storage rc err = "-1007" # probably should be -1007, revisit later err2 = "-1009" # create pool and connect self.prepare_pool() # query the pool self.log.info("Pool Query before write") self.pool.set_query_data() self.log.info("Pool %s query data: %s\n", self.pool.uuid, self.pool.query_data) # create a container try: self.log.info("creating container 1") cont = DaosContainer(self.context) cont.create(self.pool.pool.handle) self.log.info("created container 1") except DaosApiError as excep: self.log.error("caught exception creating container: " "%s", excep) self.fail("caught exception creating container: {}".format(excep)) self.log.info("opening container 1") cont.open() # generate random dkey, akey each time # write 1mb until no space, then 1kb, etc. to fill pool quickly for obj_sz in [1048576, 10240, 10, 1]: write_count = 0 while True: self.d_log.debug("writing obj {0} sz {1} to " "container".format(write_count, obj_sz)) my_str = b"a" * obj_sz my_str_sz = obj_sz dkey = get_random_bytes(5) akey = get_random_bytes(5) try: dummy_oid = cont.write_an_obj(my_str, my_str_sz, dkey, akey, obj_cls="OC_SX") self.d_log.debug("wrote obj {0}, sz {1}".format( write_count, obj_sz)) write_count += 1 except DaosApiError as excep: if not (err in repr(excep) or err2 in repr(excep)): self.log.error( "caught exception while writing " "object: %s", repr(excep)) cont.close() self.fail("caught exception while writing " "object: {}".format(repr(excep))) else: self.log.info( "pool is too full for %s byte " "objects", obj_sz) break self.log.info("closing container") cont.close() # query the pool self.log.info("Pool Query after filling") self.pool.set_query_data() self.log.info("Pool %s query data: %s\n", self.pool.uuid, self.pool.query_data) # create a 2nd container now that pool is full try: self.log.info("creating 2nd container") cont2 = DaosContainer(self.context) cont2.create(self.pool.pool.handle) self.log.info("created 2nd container") self.log.info("opening container 2") cont2.open() self.log.info("writing one more object, write expected to fail") cont2.write_an_obj(my_str, my_str_sz, dkey, akey, obj_cls="OC_SX") self.log.info("closing container") cont2.close() self.fail("wrote one more object after pool was completely filled," " this should never print") except DaosApiError as excep: if not (err in repr(excep) or err2 in repr(excep)): self.log.error( "caught unexpected exception while " "writing object: %s", repr(excep)) self.log.info("closing container") cont2.close() self.fail("caught unexpected exception while writing " "object: {}".format(repr(excep))) else: self.log.info("correctly caught -1007 while attempting " "to write object in full pool") self.log.info("closing container") cont2.close()
class OSAUtils(IorTestBase): # pylint: disable=too-many-ancestors """ Test Class Description: This test runs daos_server offline drain test cases. :avocado: recursive """ def setUp(self): """Set up for test case.""" super(OSAUtils, self).setUp() self.container = None self.obj = None self.ioreq = None self.dmg_command = self.get_dmg_command() self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*', default=[0])[0] self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*', default=[0])[0] self.record_length = self.params.get("length", '/run/record/*', default=[0])[0] @fail_on(CommandFailure) def get_pool_leader(self): """Get the pool leader. Returns: int: pool leader value """ data = self.dmg_command.pool_query(self.pool.uuid) return int(data["leader"]) @fail_on(CommandFailure) def get_rebuild_status(self): """Get the rebuild status. Returns: str: reuild status """ data = self.dmg_command.pool_query(self.pool.uuid) return data["rebuild"]["status"] @fail_on(CommandFailure) def is_rebuild_done(self, time_interval): """Rebuild is completed/done. Args: time_interval: Wait interval between checks Returns: False: If rebuild_status not "done" or "completed". True: If rebuild status is "done" or "completed". """ status = False fail_count = 0 completion_flag = ["done", "completed"] while fail_count <= 20: rebuild_status = self.get_rebuild_status() time.sleep(time_interval) fail_count += 1 if rebuild_status in completion_flag: status = True break return status @fail_on(CommandFailure) def assert_on_rebuild_failure(self): """If the rebuild is not successful, raise assert. """ rebuild_status = self.get_rebuild_status() self.log.info("Rebuild Status: %s", rebuild_status) rebuild_failed_string = ["failed", "scanning", "aborted", "busy"] self.assertTrue(rebuild_status not in rebuild_failed_string, "Rebuild failed") @fail_on(CommandFailure) def get_pool_version(self): """Get the pool version. Returns: int: pool_version_value """ data = self.dmg_command.pool_query(self.pool.uuid) return int(data["version"]) @fail_on(DaosApiError) def write_single_object(self): """Write some data to the existing pool.""" self.pool.connect(2) csum = self.params.get("enable_checksum", '/run/container/*') self.container = DaosContainer(self.context) input_param = self.container.cont_input_values input_param.enable_chksum = csum self.container.create(poh=self.pool.pool.handle, con_prop=input_param) self.container.open() self.obj = DaosObj(self.context, self.container) self.obj.create(objcls=1) self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) self.log.info("Writing the Single Dataset") for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length) d_key_value = "dkey {0}".format(dkey) c_dkey = ctypes.create_string_buffer(d_key_value) a_key_value = "akey {0}".format(akey) c_akey = ctypes.create_string_buffer(a_key_value) c_value = ctypes.create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size) self.obj.close() self.container.close() @fail_on(DaosApiError) def verify_single_object(self): """Verify the container data on the existing pool.""" self.pool.connect(2) self.container.open() self.obj.open() self.log.info("Single Dataset Verification -- Started") for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1) if indata != (repr(val.value)[1:-1]): self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR: Data mismatch for dkey = {0}, akey={1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.obj.close() self.container.close() def ior_thread(self, pool, oclass, api, test, flags, results): """Start threads and wait until all threads are finished. Args: pool (object): pool handle oclass (str): IOR object class api (str): IOR api test (list): IOR test sequence flags (str): IOR flags results (queue): queue for returning thread results """ container_info = {} mpio_util = MpioUtils() if mpio_util.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test : Mpich not installed on :" " {}".format(self.hostfile_clients[0])) self.pool = pool # Define the arguments for the ior_runner_thread method ior_cmd = IorCommand() ior_cmd.get_params(self) ior_cmd.set_daos_params(self.server_group, self.pool) ior_cmd.dfs_oclass.update(oclass) ior_cmd.api.update(api) ior_cmd.transfer_size.update(test[2]) ior_cmd.block_size.update(test[3]) ior_cmd.flags.update(flags) container_info["{}{}{}".format(oclass, api, test[2])] = str(uuid.uuid4()) # Define the job manager for the IOR command self.job_manager = Mpirun(ior_cmd, mpitype="mpich") key = "".join([oclass, api, str(test[2])]) self.job_manager.job.dfs_cont.update(container_info[key]) env = ior_cmd.get_default_env(str(self.job_manager)) self.job_manager.assign_hosts(self.hostlist_clients, self.workdir, None) self.job_manager.assign_processes(self.processes) self.job_manager.assign_environment(env, True) # run IOR Command try: self.job_manager.run() except CommandFailure as _error: results.put("FAIL")
def test_bad_handle(self): """ Test ID: DAOS-1376 Test Description: Pass a bogus object handle, should return bad handle. :avocado: tags=all,object,full_regression,small,objbadhand """ try: # parameters used in pool create createmode = self.params.get("mode", '/run/conttests/createmode/') createsetid = self.params.get("setname", '/run/conttests/createset/') createsize = self.params.get("size", '/run/conttests/createsize/') createuid = os.geteuid() creategid = os.getegid() # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) self.plog.info("Pool %s created.", pool.get_uuid_str()) # need a connection to create container pool.connect(1 << 1) # create a container container = DaosContainer(self.context) container.create(pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # create an object and write some data into it thedata = "a string that I want to stuff into an object" thedatasize = len(thedata) + 1 dkey = "this is the dkey" akey = "this is the akey" obj, dummy_tx = container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) saved_oh = obj.obj_handle obj.obj_handle = 99999 obj, dummy_tx = container.write_an_obj(thedata, thedatasize, dkey, akey, obj, None, 2) container.oh = saved_oh container.close() container.destroy() pool.disconnect() pool.destroy(1) self.fail("Test was expected to return a -1002 but it has not.\n") except DaosApiError as excep: container.oh = saved_oh container.close() container.destroy() pool.disconnect() pool.destroy(1) self.plog.info("Test Complete") if '-1002' not in str(excep): print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1002 but it has not.\n")
class ObjectDataValidation(TestWithServers): """ Test Class Description: Tests that create Different length records, Disconnect the pool/container and reconnect, validate the data after reconnect. :avocado: recursive """ # pylint: disable=too-many-instance-attributes def setUp(self): super(ObjectDataValidation, self).setUp() self.obj = None self.ioreq = None self.no_of_dkeys = None self.no_of_akeys = None self.array_size = None self.record_length = None self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0] self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0] self.array_size = self.params.get("size", '/array_size/') self.record_length = self.params.get("length", '/run/record/*') self.prepare_pool() self.container = DaosContainer(self.context) self.container.create(self.pool.pool.handle) self.container.open() self.obj = DaosObj(self.context, self.container) self.obj.create(objcls=1) self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) def reconnect(self): ''' Function to reconnect the pool/container and reopen the Object for read verification. ''' #Close the Obj/Container, Disconnect the Pool. self.obj.close() self.container.close() self.pool.disconnect() time.sleep(5) #Connect Pool, Open Container and Object self.pool.connect(2) self.container.open() self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) @avocado.fail_on(DaosApiError) def test_invalid_tx_commit_close(self): """ Test ID: (1)DAOS-1346: Verify commit tx bad parameter behavior. (2)DAOS-1343: Verify tx_close bad parameter behavior. (3)DAOS-1342: Verify tx_close through daos_api. (4)DAOS-1338: Add and verify tx_abort through daos_api. (5)DAOS-1339: Verify tx_abort bad parameter behavior. Test Description: Write Avocado Test to verify commit tx and close tx bad parameter behavior. :avocado: tags=all,object,full_regression,small,invalid_tx """ self.d_log.info("==Writing the Single Dataset for negative test...") record_index = 0 expected_error = "RC: -1002" dkey = 0 akey = 0 indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_value = ctypes.create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) try: new_transaction = self.container.get_new_tx() except DaosApiError as excep: #initial container get_new_tx failed, skip rest of the test self.fail("##container get_new_tx failed: {}".format(excep)) invalid_transaction = new_transaction + random.randint(1000, 383838) self.log.info("==new_transaction= %s", new_transaction) self.log.info("==invalid_transaction= %s", invalid_transaction) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size, new_transaction) try: self.container.commit_tx(invalid_transaction) self.fail("##(1.1)Container.commit_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(1)Expecting failure: invalid Container.commit_tx.") if expected_error not in str(excep): self.fail( "##(1.2)Expecting error RC: -1002, but got {}.".format( str(excep))) try: self.container.close_tx(invalid_transaction) self.fail("##(2.1)Container.close_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(2)Expecting failure: invalid Container.commit_tx.") if expected_error not in str(excep): self.fail( "##(2.2)Expecting error RC: -1002, but got {}.".format( str(excep))) try: self.container.close_tx(new_transaction) self.log.info("==(3)container.close_tx test passed.") except DaosApiError as excep: self.log.info(str(excep)) self.fail("##(3)Failed on close_tx.") try: self.container.abort_tx(invalid_transaction) self.fail("##(4.1)Container.abort_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(4)Expecting failure: invalid Container.abort_tx.") if expected_error not in str(excep): self.fail( "##(4.2)Expecting error RC: -1002, but got {}.".format( str(excep))) #Try to abort the transaction which already closed. try: self.container.abort_tx(new_transaction) self.fail("##(5.1)Container.abort_tx passing with a closed handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(5)Expecting failure: Container.abort_tx closed handle.") if expected_error not in str(excep): self.fail( "##(5.2)Expecting error RC: -1002, but got {}.".format( str(excep))) #open another transaction for abort test try: new_transaction2 = self.container.get_new_tx() except DaosApiError as excep: self.fail("##(6.1)container get_new_tx failed: {}".format(excep)) self.log.info("==new_transaction2= %s", new_transaction2) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size, new_transaction2) try: self.container.abort_tx(new_transaction2) self.log.info("==(6)container.abort_tx test passed.") except DaosApiError as excep: self.log.info(str(excep)) self.fail("##(6.2)Failed on abort_tx.") self.container.close_tx(new_transaction2) @avocado.fail_on(DaosApiError) def test_single_object_validation(self): """ Test ID: DAOS-707 Test Description: Write Avocado Test to verify single data after pool/container disconnect/reconnect. :avocado: tags=all,object,full_regression,small,single_object """ self.d_log.info("Writing the Single Dataset") record_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_value = ctypes.create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size) record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 self.reconnect() self.d_log.info("Single Dataset Verification -- Started") record_index = 0 transaction_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1) if indata != (repr(val.value)[1:-1]): self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR: Data mismatch for dkey = {0}, akey={1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) transaction_index = transaction_index + 1 record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 @avocado.fail_on(DaosApiError) def test_array_object_validation(self): """ Test ID: DAOS-707 Test Description: Write Avocado Test to verify Array data after pool/container disconnect/reconnect. :avocado: tags=all,object,full_regression,small,array_object """ self.d_log.info("Writing the Array Dataset") record_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): c_values = [] value = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) for item in range(self.array_size): c_values.append( (ctypes.create_string_buffer(value), len(value) + 1)) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) self.ioreq.insert_array(c_dkey, c_akey, c_values) record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 self.reconnect() self.d_log.info("Array Dataset Verification -- Started") record_index = 0 transaction_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = [] value = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) for item in range(self.array_size): indata.append(value) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_rec_count = ctypes.c_uint(len(indata)) c_rec_size = ctypes.c_size_t(len(indata[0]) + 1) outdata = self.ioreq.fetch_array(c_dkey, c_akey, c_rec_count, c_rec_size) for item in enumerate(indata): if indata[item[0]] != outdata[item[0]][:-1]: self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR:Data mismatch for dkey = {0}, akey={1}". format("dkey {0}".format(dkey), "akey {0}".format(akey))) transaction_index = transaction_index + 1 record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0
class ObjOpenBadParam(TestWithServers): """ Test Class Description: Pass an assortment of bad parameters to the daos_obj_open function. :avocado: recursive """ def setUp(self): super(ObjOpenBadParam, self).setUp() self.prepare_pool() try: # create a container self.container = DaosContainer(self.context) self.container.create(self.pool.pool.handle) # now open it self.container.open() # create an object and write some data into it thedata = "a string that I want to stuff into an object" self.datasize = len(thedata) + 1 self.dkey = "this is the dkey" self.akey = "this is the akey" self.obj = self.container.write_an_obj(thedata, self.datasize, self.dkey, self.akey, obj_cls=1) thedata2 = self.container.read_an_obj(self.datasize, self.dkey, self.akey, self.obj) if thedata not in thedata2.value: print(thedata) print(thedata2.value) err_str = "Error reading back data, test failed during the " \ "initial setup." self.d_log.error(err_str) self.fail(err_str) # setup leaves object in open state, so closing to start clean self.obj.close() except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test failed during the initial setup.") def test_bad_obj_handle(self): """ Test ID: DAOS-1320 Test Description: Attempt to open a garbage object handle. :avocado: tags=all,object,full_regression,tiny,objopenbadhandle """ saved_handle = self.obj.obj_handle self.obj.obj_handle = 8675309 try: dummy_obj = self.obj.open() except DaosApiError as excep: if '-1002' not in str(excep): self.d_log.error("test expected a -1002 but did not get it") self.d_log.error(traceback.format_exc()) self.fail("test expected a -1002 but did not get it") finally: self.obj.obj_handle = saved_handle def test_invalid_container_handle(self): """ Test ID: DAOS-1320 Test Description: Attempt to open an object with a garbage container handle. :avocado: tags=all,object,full_regression,tiny,objopenbadcont """ saved_coh = self.container.coh self.container.coh = 8675309 try: dummy_obj = self.obj.open() except DaosApiError as excep: if '-1002' not in str(excep): self.d_log.error("test expected a -1002 but did not get it") self.d_log.error(traceback.format_exc()) self.fail("test expected a -1002 but did not get it") finally: self.container.coh = saved_coh def test_closed_container_handle(self): """ Test ID: DAOS-1320 Test Description: Attempt to open an object in a container with a closed handle. :avocado: tags=all,object,full_regression,tiny,objopenclosedcont """ self.container.close() try: dummy_obj = self.obj.open() except DaosApiError as excep: if '-1002' not in str(excep): self.d_log.error("test expected a -1002 but did not get it") self.d_log.error(traceback.format_exc()) self.fail("test expected a -1002 but did not get it") finally: self.container.open() def test_pool_handle_as_obj_handle(self): """ Test ID: DAOS-1320 Test Description: Adding this test by request, this test attempts to open an object that's had its handle set to be the same as a valid pool handle. :avocado: tags=all,object,full_regression,tiny,objopenbadpool """ saved_oh = self.obj.obj_handle self.obj.obj_handle = self.pool.pool.handle try: dummy_obj = self.obj.open() except DaosApiError as excep: if '-1002' not in str(excep): self.d_log.error("test expected a -1002 but did not get it") self.d_log.error(traceback.format_exc()) self.fail("test expected a -1002 but did not get it") finally: self.obj.obj_handle = saved_oh def test_null_ranklist(self): """ Test ID: DAOS-1320 Test Description: Attempt to open an object in a container with an empty ranklist. :avocado: tags=all,object,full_regression,tiny,objopennullrl """ # null rl saved_rl = self.obj.tgt_rank_list self.obj.tgt_rank_list = None try: dummy_obj = self.obj.open() except DaosApiError as excep: if '-1003' not in str(excep): self.d_log.error("test expected a -1003 but did not get it") self.d_log.error(traceback.format_exc()) self.fail("test expected a -1003 but did not get it") finally: self.obj.tgt_rank_list = saved_rl def test_null_oid(self): """ Test ID: DAOS-1320 Test Description: Attempt to open an object in a container with null object id. :avocado: tags=all,object,full_regression,tiny,objopennulloid """ # null oid saved_oid = self.obj.c_oid self.obj.c_oid = DaosObjId(0, 0) try: dummy_obj = self.obj.open() except DaosApiError as excep: if '-1003' not in str(excep): self.d_log.error("Test expected a -1003 but did not get it") self.d_log.error(traceback.format_exc()) self.fail("test expected a -1003 but did not get it") finally: self.obj.c_oid = saved_oid def test_null_tgts(self): """ Test ID: DAOS-1320 Test Description: Attempt to open an object in a container with null tgt. :avocado: tags=all,object,full_regression,tiny,objopennulltgts """ # null tgts saved_ctgts = self.obj.c_tgts self.obj.c_tgts = 0 try: dummy_obj = self.obj.open() except DaosApiError as excep: if '-1003' not in str(excep): self.d_log.error("Test expected a -1003 but did not get it") self.d_log.error(traceback.format_exc()) self.fail("test expected a -1003 but did not get it") finally: self.obj.c_tgts = saved_ctgts def test_null_attrs(self): """ Test ID: DAOS-1320 Test Description: Attempt to open an object in a container with null object attributes. :avocado: tags=all,object,full_regression,tiny,objopennullattr """ # null attr saved_attr = self.obj.attr self.obj.attr = 0 try: dummy_obj = self.obj.open() except DaosApiError as excep: if '-1003' not in str(excep): self.d_log.error("test expected a -1003 but did not get it") self.d_log.error(traceback.format_exc()) self.fail("test expected a -1003 but did not get it") finally: self.obj.attr = saved_attr
def test_array_obj(self): """ Test ID: DAOS-961 Test Description: Writes an array to an object and then reads it back and verifies it. :avocado: tags=all,smoke,pr,object,tiny,basicobject """ self.prepare_pool() try: # create a container container = DaosContainer(self.context) container.create(self.pool.pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # do a query and compare the UUID returned from create with # that returned by query container.query() if container.get_uuid_str() != c_uuid_to_str( container.info.ci_uuid): self.fail("Container UUID did not match the one in info\n") # create an object and write some data into it thedata = [] thedata.append("data string one") thedata.append("data string two") thedata.append("data string tre") dkey = "this is the dkey" akey = "this is the akey" self.plog.info("writing array to dkey >%s< akey >%s<.", dkey, akey) oid = container.write_an_array_value(thedata, dkey, akey, obj_cls=3) # read the data back and make sure its correct length = len(thedata[0]) thedata2 = container.read_an_array(len(thedata), length + 1, dkey, akey, oid) if thedata[0][0:length - 1] != thedata2[0][0:length - 1]: self.plog.error("Data mismatch") self.plog.error("Wrote: >%s<", thedata[0]) self.plog.error("Read: >%s<", thedata2[0]) self.fail("Write data, read it back, didn't match\n") if thedata[2][0:length - 1] != thedata2[2][0:length - 1]: self.plog.error("Data mismatch") self.plog.error("Wrote: >%s<", thedata[2]) self.plog.error("Read: >%s<", thedata2[2]) self.fail("Write data, read it back, didn't match\n") container.close() # wait a few seconds and then destroy time.sleep(5) container.destroy() self.plog.info("Test Complete") except DaosApiError as excep: self.plog.error("Test Failed, exception was thrown.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to pass but it failed.\n")
class ObjFetchBadParam(TestWithServers): """ Test Class Description: Pass an assortment of bad parameters to the daos_obj_fetch function. :avocado: recursive """ def setUp(self): super(ObjFetchBadParam, self).setUp() time.sleep(5) self.prepare_pool() try: # create a container self.container = DaosContainer(self.context) self.container.create(self.pool.pool.handle) # now open it self.container.open() # create an object and write some data into it thedata = "a string that I want to stuff into an object" self.datasize = len(thedata) + 1 self.dkey = "this is the dkey" self.akey = "this is the akey" self.obj = self.container.write_an_obj(thedata, self.datasize, self.dkey, self.akey, None, None, 2) thedata2 = self.container.read_an_obj(self.datasize, self.dkey, self.akey, self.obj) if thedata not in thedata2.value: print(thedata) print(thedata2.value) self.fail("Error reading back data, test failed during"\ " the initial setup.\n") except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test failed during the initial setup.\n") def test_bad_handle(self): """ Test ID: DAOS-1377 Test Description: Pass a bogus object handle, should return bad handle. :avocado: tags=all,object,full_regression,small,objbadhandle """ try: # trash the handle and read again saved_oh = self.obj.obj_handle self.obj.obj_handle = 99999 # expecting this to fail with -1002 dummy_thedata2 = self.container.read_an_obj( self.datasize, self.dkey, self.akey, self.obj) self.container.oh = saved_oh self.fail("Test was expected to return a -1002 but it has not.\n") except DaosApiError as excep: self.container.oh = saved_oh if '-1002' not in str(excep): print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1002 but it has not.\n") def test_null_ptrs(self): """ Test ID: DAOS-1377 Test Description: Pass null pointers for various fetch parameters. :avocado: tags=all,object,full_regression,small,objfetchnull """ try: # now try it with a bad dkey, expecting this to fail with -1003 dummy_thedata2 = self.container.read_an_obj( self.datasize, None, self.akey, self.obj) self.container.close() self.container.destroy() self.pool.destroy(1) self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # now try it with a null sgl (iod_size is not set) # expecting this to fail with -2013 test_hints = ['sglnull'] dummy_thedata2 = self.container.read_an_obj( self.datasize, self.dkey, self.akey, self.obj, test_hints) # behavior not as expect so commented out for now # when DAOS-1448 is complete, uncomment and retest self.fail("Test was expected to return a -2013 but it has not.\n") except DaosApiError as excep: if '-2013' not in str(excep): print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -2013 but it has not.\n") try: # when DAOS-1449 is complete, uncomment and retest # now try it with a null iod, expecting this to fail with -1003 #test_hints = ['iodnull'] #thedata2 = self.container.read_an_obj(self.datasize, dkey, akey, # self.obj, test_hints) pass #self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n")
class ObjectDataValidation(avocado.Test): """ Test Class Description: Tests that create Different length records, Disconnect the pool/container and reconnect, validate the data after reconnect. """ # pylint: disable=too-many-instance-attributes def setUp(self): self.agent_sessions = None self.pool = None self.container = None self.obj = None self.ioreq = None self.hostlist = None self.hostfile = None self.no_of_dkeys = None self.no_of_akeys = None self.array_size = None self.record_length = None with open('../../.build_vars.json') as json_f: build_paths = json.load(json_f) self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../") server_group = self.params.get("name", '/server_config/', 'daos_server') self.context = DaosContext(build_paths['PREFIX'] + '/lib64/') self.d_log = DaosLog(self.context) self.hostlist = self.params.get("test_machines", '/run/hosts/*') self.hostfile = write_host_file.write_host_file( self.hostlist, self.workdir) self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0] self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0] self.array_size = self.params.get("size", '/array_size/') self.record_length = self.params.get("length", '/run/record/*') self.agent_sessions = agent_utils.run_agent(self.basepath, self.hostlist) server_utils.run_server(self, self.hostfile, server_group) self.pool = DaosPool(self.context) self.pool.create(self.params.get("mode", '/run/pool/createmode/*'), os.geteuid(), os.getegid(), self.params.get("size", '/run/pool/createsize/*'), self.params.get("setname", '/run/pool/createset/*'), None) self.pool.connect(2) self.container = DaosContainer(self.context) self.container.create(self.pool.handle) self.container.open() self.obj = DaosObj(self.context, self.container) self.obj.create(objcls=1) self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) def tearDown(self): try: if self.container: self.container.close() self.container.destroy() if self.pool: self.pool.disconnect() self.pool.destroy(1) finally: if self.agent_sessions: agent_utils.stop_agent(self.agent_sessions) server_utils.stop_server(hosts=self.hostlist) def reconnect(self): ''' Function to reconnect the pool/container and reopen the Object for read verification. ''' #Close the Obj/Container, Disconnect the Pool. self.obj.close() self.container.close() self.pool.disconnect() time.sleep(5) #Connect Pool, Open Container and Object self.pool.connect(2) self.container.open() self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) @avocado.fail_on(DaosApiError) def test_invalid_tx_commit_close(self): """ Test ID: (1)DAOS-1346: Verify commit tx bad parameter behavior. (2)DAOS-1343: Verify tx_close bad parameter behavior. (3)DAOS-1342: Verify tx_close through daos_api. (4)DAOS-1338: Add and verify tx_abort through daos_api. (5)DAOS-1339: Verify tx_abort bad parameter behavior. Test Description: Write Avocado Test to verify commit tx and close tx bad parameter behavior. :avocado: tags=all,object,full_regression,small,invalid_tx """ self.d_log.info("==Writing the Single Dataset for negative test...") record_index = 0 expected_error = "RC: -1002" dkey = 0 akey = 0 indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_value = ctypes.create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) try: new_transaction = self.container.get_new_tx() except DaosApiError as excep: #initial container get_new_tx failed, skip rest of the test self.fail("##container get_new_tx failed: {}".format(excep)) invalid_transaction = new_transaction + random.randint(1000, 383838) self.log.info("==new_transaction= %s", new_transaction) self.log.info("==invalid_transaction= %s", invalid_transaction) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size, new_transaction) try: self.container.commit_tx(invalid_transaction) self.fail("##(1.1)Container.commit_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(1)Expecting failure: invalid Container.commit_tx.") if expected_error not in str(excep): self.fail( "##(1.2)Expecting error RC: -1002, but got {}.".format( str(excep))) try: self.container.close_tx(invalid_transaction) self.fail("##(2.1)Container.close_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(2)Expecting failure: invalid Container.commit_tx.") if expected_error not in str(excep): self.fail( "##(2.2)Expecting error RC: -1002, but got {}.".format( str(excep))) try: self.container.close_tx(new_transaction) self.log.info("==(3)container.close_tx test passed.") except DaosApiError as excep: self.log.info(str(excep)) self.fail("##(3)Failed on close_tx.") try: self.container.abort_tx(invalid_transaction) self.fail("##(4.1)Container.abort_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(4)Expecting failure: invalid Container.abort_tx.") if expected_error not in str(excep): self.fail( "##(4.2)Expecting error RC: -1002, but got {}.".format( str(excep))) #Try to abort the transaction which already closed. try: self.container.abort_tx(new_transaction) self.fail("##(5.1)Container.abort_tx passing with a closed handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(5)Expecting failure: Container.abort_tx closed handle.") if expected_error not in str(excep): self.fail( "##(5.2)Expecting error RC: -1002, but got {}.".format( str(excep))) #open another transaction for abort test try: new_transaction2 = self.container.get_new_tx() except DaosApiError as excep: self.fail("##(6.1)container get_new_tx failed: {}".format(excep)) self.log.info("==new_transaction2= %s", new_transaction2) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size, new_transaction2) try: self.container.abort_tx(new_transaction2) self.log.info("==(6)container.abort_tx test passed.") except DaosApiError as excep: self.log.info(str(excep)) self.fail("##(6.2)Failed on abort_tx.") @avocado.fail_on(DaosApiError) @skipForTicket("DAOS-3208") def test_single_object_validation(self): """ Test ID: DAOS-707 Test Description: Write Avocado Test to verify single data after pool/container disconnect/reconnect. :avocado: tags=all,object,full_regression,small,single_object """ self.d_log.info("Writing the Single Dataset") record_index = 0 transaction = [] for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_value = ctypes.create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) new_transaction = self.container.get_new_tx() self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size, new_transaction) self.container.commit_tx(new_transaction) transaction.append(new_transaction) record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 self.reconnect() self.d_log.info("Single Dataset Verification -- Started") record_index = 0 transaction_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1) if indata != (repr(val.value)[1:-1]): self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR: Data mismatch for dkey = {0}, akey={1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) transaction_index = transaction_index + 1 record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 @avocado.fail_on(DaosApiError) @skipForTicket("DAOS-3208") def test_array_object_validation(self): """ Test ID: DAOS-707 Test Description: Write Avocado Test to verify Array data after pool/container disconnect/reconnect. :avocado: tags=all,object,full_regression,small,array_object """ self.d_log.info("Writing the Array Dataset") record_index = 0 transaction = [] for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): c_values = [] value = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) for item in range(self.array_size): c_values.append( (ctypes.create_string_buffer(value), len(value) + 1)) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) new_transaction = self.container.get_new_tx() self.ioreq.insert_array(c_dkey, c_akey, c_values, new_transaction) self.container.commit_tx(new_transaction) transaction.append(new_transaction) record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 self.reconnect() self.d_log.info("Array Dataset Verification -- Started") record_index = 0 transaction_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = [] value = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) for item in range(self.array_size): indata.append(value) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_rec_count = ctypes.c_uint(len(indata)) c_rec_size = ctypes.c_size_t(len(indata[0]) + 1) outdata = self.ioreq.fetch_array(c_dkey, c_akey, c_rec_count, c_rec_size) for item in enumerate(indata): if indata[item[0]] != outdata[item[0]][:-1]: self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR:Data mismatch for dkey = {0}, akey={1}". format("dkey {0}".format(dkey), "akey {0}".format(akey))) transaction_index = transaction_index + 1 record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0
class OSAUtils(TestWithServers): # pylint: disable=too-many-ancestors """ Test Class Description: This test runs daos_server offline drain test cases. :avocado: recursive """ def setUp(self): """Set up for test case.""" super(OSAUtils, self).setUp() self.container = None self.obj = None self.ioreq = None self.dmg_command = self.get_dmg_command() self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0] self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0] self.record_length = self.params.get("length", '/run/record/*')[0] @fail_on(CommandFailure) def get_pool_leader(self): """Get the pool leader. Returns: int: pool leader value """ data = self.dmg_command.pool_query(self.pool.uuid) return int(data["leader"]) @fail_on(CommandFailure) def get_pool_version(self): """Get the pool version. Returns: int: pool_version_value """ data = self.dmg_command.pool_query(self.pool.uuid) return int(data["version"]) @fail_on(DaosApiError) def write_single_object(self): """Write some data to the existing pool.""" self.pool.connect(2) csum = self.params.get("enable_checksum", '/run/container/*') self.container = DaosContainer(self.context) input_param = self.container.cont_input_values input_param.enable_chksum = csum self.container.create(poh=self.pool.pool.handle, con_prop=input_param) self.container.open() self.obj = DaosObj(self.context, self.container) self.obj.create(objcls=1) self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) self.log.info("Writing the Single Dataset") for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length) d_key_value = "dkey {0}".format(dkey) c_dkey = ctypes.create_string_buffer(d_key_value) a_key_value = "akey {0}".format(akey) c_akey = ctypes.create_string_buffer(a_key_value) c_value = ctypes.create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size) self.obj.close() self.container.close() @fail_on(DaosApiError) def verify_single_object(self): """Verify the container data on the existing pool.""" self.pool.connect(2) self.container.open() self.obj.open() self.log.info("Single Dataset Verification -- Started") for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1) if indata != (repr(val.value)[1:-1]): self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR: Data mismatch for dkey = {0}, akey={1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.obj.close() self.container.close()