class ObjectDataValidation(TestWithServers): """ Test Class Description: Tests that create Different length records, Disconnect the pool/container and reconnect, validate the data after reconnect. :avocado: recursive """ # pylint: disable=too-many-instance-attributes def setUp(self): super(ObjectDataValidation, self).setUp() self.obj = None self.ioreq = None self.no_of_dkeys = None self.no_of_akeys = None self.array_size = None self.record_length = None self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0] self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0] self.array_size = self.params.get("size", '/array_size/') self.record_length = self.params.get("length", '/run/record/*') self.prepare_pool() self.container = DaosContainer(self.context) self.container.create(self.pool.pool.handle) self.container.open() self.obj = DaosObj(self.context, self.container) self.obj.create(objcls=1) self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) def reconnect(self): ''' Function to reconnect the pool/container and reopen the Object for read verification. ''' #Close the Obj/Container, Disconnect the Pool. self.obj.close() self.container.close() self.pool.disconnect() time.sleep(5) #Connect Pool, Open Container and Object self.pool.connect(2) self.container.open() self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) @avocado.fail_on(DaosApiError) def test_invalid_tx_commit_close(self): """ Test ID: (1)DAOS-1346: Verify commit tx bad parameter behavior. (2)DAOS-1343: Verify tx_close bad parameter behavior. (3)DAOS-1342: Verify tx_close through daos_api. (4)DAOS-1338: Add and verify tx_abort through daos_api. (5)DAOS-1339: Verify tx_abort bad parameter behavior. Test Description: Write Avocado Test to verify commit tx and close tx bad parameter behavior. :avocado: tags=all,object,full_regression,small,invalid_tx """ self.d_log.info("==Writing the Single Dataset for negative test...") record_index = 0 expected_error = "RC: -1002" dkey = 0 akey = 0 indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_value = ctypes.create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) try: new_transaction = self.container.get_new_tx() except DaosApiError as excep: #initial container get_new_tx failed, skip rest of the test self.fail("##container get_new_tx failed: {}".format(excep)) invalid_transaction = new_transaction + random.randint(1000, 383838) self.log.info("==new_transaction= %s", new_transaction) self.log.info("==invalid_transaction= %s", invalid_transaction) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size, new_transaction) try: self.container.commit_tx(invalid_transaction) self.fail("##(1.1)Container.commit_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(1)Expecting failure: invalid Container.commit_tx.") if expected_error not in str(excep): self.fail( "##(1.2)Expecting error RC: -1002, but got {}.".format( str(excep))) try: self.container.close_tx(invalid_transaction) self.fail("##(2.1)Container.close_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(2)Expecting failure: invalid Container.commit_tx.") if expected_error not in str(excep): self.fail( "##(2.2)Expecting error RC: -1002, but got {}.".format( str(excep))) try: self.container.close_tx(new_transaction) self.log.info("==(3)container.close_tx test passed.") except DaosApiError as excep: self.log.info(str(excep)) self.fail("##(3)Failed on close_tx.") try: self.container.abort_tx(invalid_transaction) self.fail("##(4.1)Container.abort_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(4)Expecting failure: invalid Container.abort_tx.") if expected_error not in str(excep): self.fail( "##(4.2)Expecting error RC: -1002, but got {}.".format( str(excep))) #Try to abort the transaction which already closed. try: self.container.abort_tx(new_transaction) self.fail("##(5.1)Container.abort_tx passing with a closed handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(5)Expecting failure: Container.abort_tx closed handle.") if expected_error not in str(excep): self.fail( "##(5.2)Expecting error RC: -1002, but got {}.".format( str(excep))) #open another transaction for abort test try: new_transaction2 = self.container.get_new_tx() except DaosApiError as excep: self.fail("##(6.1)container get_new_tx failed: {}".format(excep)) self.log.info("==new_transaction2= %s", new_transaction2) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size, new_transaction2) try: self.container.abort_tx(new_transaction2) self.log.info("==(6)container.abort_tx test passed.") except DaosApiError as excep: self.log.info(str(excep)) self.fail("##(6.2)Failed on abort_tx.") self.container.close_tx(new_transaction2) @avocado.fail_on(DaosApiError) def test_single_object_validation(self): """ Test ID: DAOS-707 Test Description: Write Avocado Test to verify single data after pool/container disconnect/reconnect. :avocado: tags=all,object,full_regression,small,single_object """ self.d_log.info("Writing the Single Dataset") record_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_value = ctypes.create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size) record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 self.reconnect() self.d_log.info("Single Dataset Verification -- Started") record_index = 0 transaction_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1) if indata != (repr(val.value)[1:-1]): self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR: Data mismatch for dkey = {0}, akey={1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) transaction_index = transaction_index + 1 record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 @avocado.fail_on(DaosApiError) def test_array_object_validation(self): """ Test ID: DAOS-707 Test Description: Write Avocado Test to verify Array data after pool/container disconnect/reconnect. :avocado: tags=all,object,full_regression,small,array_object """ self.d_log.info("Writing the Array Dataset") record_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): c_values = [] value = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) for item in range(self.array_size): c_values.append( (ctypes.create_string_buffer(value), len(value) + 1)) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) self.ioreq.insert_array(c_dkey, c_akey, c_values) record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 self.reconnect() self.d_log.info("Array Dataset Verification -- Started") record_index = 0 transaction_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = [] value = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) for item in range(self.array_size): indata.append(value) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_rec_count = ctypes.c_uint(len(indata)) c_rec_size = ctypes.c_size_t(len(indata[0]) + 1) outdata = self.ioreq.fetch_array(c_dkey, c_akey, c_rec_count, c_rec_size) for item in enumerate(indata): if indata[item[0]] != outdata[item[0]][:-1]: self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR:Data mismatch for dkey = {0}, akey={1}". format("dkey {0}".format(dkey), "akey {0}".format(akey))) transaction_index = transaction_index + 1 record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0
class ObjectDataValidation(avocado.Test): """ Test Class Description: Tests that create Different length records, Disconnect the pool/container and reconnect, validate the data after reconnect. """ # pylint: disable=too-many-instance-attributes def setUp(self): self.agent_sessions = None self.pool = None self.container = None self.obj = None self.ioreq = None self.hostlist = None self.hostfile = None self.no_of_dkeys = None self.no_of_akeys = None self.array_size = None self.record_length = None with open('../../.build_vars.json') as json_f: build_paths = json.load(json_f) self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../") server_group = self.params.get("name", '/server_config/', 'daos_server') self.context = DaosContext(build_paths['PREFIX'] + '/lib64/') self.d_log = DaosLog(self.context) self.hostlist = self.params.get("test_machines", '/run/hosts/*') self.hostfile = write_host_file.write_host_file( self.hostlist, self.workdir) self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0] self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0] self.array_size = self.params.get("size", '/array_size/') self.record_length = self.params.get("length", '/run/record/*') self.agent_sessions = agent_utils.run_agent(self.basepath, self.hostlist) server_utils.run_server(self, self.hostfile, server_group) self.pool = DaosPool(self.context) self.pool.create(self.params.get("mode", '/run/pool/createmode/*'), os.geteuid(), os.getegid(), self.params.get("size", '/run/pool/createsize/*'), self.params.get("setname", '/run/pool/createset/*'), None) self.pool.connect(2) self.container = DaosContainer(self.context) self.container.create(self.pool.handle) self.container.open() self.obj = DaosObj(self.context, self.container) self.obj.create(objcls=1) self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) def tearDown(self): try: if self.container: self.container.close() self.container.destroy() if self.pool: self.pool.disconnect() self.pool.destroy(1) finally: if self.agent_sessions: agent_utils.stop_agent(self.agent_sessions) server_utils.stop_server(hosts=self.hostlist) def reconnect(self): ''' Function to reconnect the pool/container and reopen the Object for read verification. ''' #Close the Obj/Container, Disconnect the Pool. self.obj.close() self.container.close() self.pool.disconnect() time.sleep(5) #Connect Pool, Open Container and Object self.pool.connect(2) self.container.open() self.obj.open() self.ioreq = IORequest(self.context, self.container, self.obj, objtype=4) @avocado.fail_on(DaosApiError) def test_invalid_tx_commit_close(self): """ Test ID: (1)DAOS-1346: Verify commit tx bad parameter behavior. (2)DAOS-1343: Verify tx_close bad parameter behavior. (3)DAOS-1342: Verify tx_close through daos_api. (4)DAOS-1338: Add and verify tx_abort through daos_api. (5)DAOS-1339: Verify tx_abort bad parameter behavior. Test Description: Write Avocado Test to verify commit tx and close tx bad parameter behavior. :avocado: tags=all,object,full_regression,small,invalid_tx """ self.d_log.info("==Writing the Single Dataset for negative test...") record_index = 0 expected_error = "RC: -1002" dkey = 0 akey = 0 indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_value = ctypes.create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) try: new_transaction = self.container.get_new_tx() except DaosApiError as excep: #initial container get_new_tx failed, skip rest of the test self.fail("##container get_new_tx failed: {}".format(excep)) invalid_transaction = new_transaction + random.randint(1000, 383838) self.log.info("==new_transaction= %s", new_transaction) self.log.info("==invalid_transaction= %s", invalid_transaction) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size, new_transaction) try: self.container.commit_tx(invalid_transaction) self.fail("##(1.1)Container.commit_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(1)Expecting failure: invalid Container.commit_tx.") if expected_error not in str(excep): self.fail( "##(1.2)Expecting error RC: -1002, but got {}.".format( str(excep))) try: self.container.close_tx(invalid_transaction) self.fail("##(2.1)Container.close_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(2)Expecting failure: invalid Container.commit_tx.") if expected_error not in str(excep): self.fail( "##(2.2)Expecting error RC: -1002, but got {}.".format( str(excep))) try: self.container.close_tx(new_transaction) self.log.info("==(3)container.close_tx test passed.") except DaosApiError as excep: self.log.info(str(excep)) self.fail("##(3)Failed on close_tx.") try: self.container.abort_tx(invalid_transaction) self.fail("##(4.1)Container.abort_tx passing with invalid handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(4)Expecting failure: invalid Container.abort_tx.") if expected_error not in str(excep): self.fail( "##(4.2)Expecting error RC: -1002, but got {}.".format( str(excep))) #Try to abort the transaction which already closed. try: self.container.abort_tx(new_transaction) self.fail("##(5.1)Container.abort_tx passing with a closed handle") except DaosApiError as excep: self.log.info(str(excep)) self.log.info( "==(5)Expecting failure: Container.abort_tx closed handle.") if expected_error not in str(excep): self.fail( "##(5.2)Expecting error RC: -1002, but got {}.".format( str(excep))) #open another transaction for abort test try: new_transaction2 = self.container.get_new_tx() except DaosApiError as excep: self.fail("##(6.1)container get_new_tx failed: {}".format(excep)) self.log.info("==new_transaction2= %s", new_transaction2) self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size, new_transaction2) try: self.container.abort_tx(new_transaction2) self.log.info("==(6)container.abort_tx test passed.") except DaosApiError as excep: self.log.info(str(excep)) self.fail("##(6.2)Failed on abort_tx.") @avocado.fail_on(DaosApiError) @skipForTicket("DAOS-3208") def test_single_object_validation(self): """ Test ID: DAOS-707 Test Description: Write Avocado Test to verify single data after pool/container disconnect/reconnect. :avocado: tags=all,object,full_regression,small,single_object """ self.d_log.info("Writing the Single Dataset") record_index = 0 transaction = [] for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_value = ctypes.create_string_buffer(indata) c_size = ctypes.c_size_t(ctypes.sizeof(c_value)) new_transaction = self.container.get_new_tx() self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size, new_transaction) self.container.commit_tx(new_transaction) transaction.append(new_transaction) record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 self.reconnect() self.d_log.info("Single Dataset Verification -- Started") record_index = 0 transaction_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1) if indata != (repr(val.value)[1:-1]): self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR: Data mismatch for dkey = {0}, akey={1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) transaction_index = transaction_index + 1 record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 @avocado.fail_on(DaosApiError) @skipForTicket("DAOS-3208") def test_array_object_validation(self): """ Test ID: DAOS-707 Test Description: Write Avocado Test to verify Array data after pool/container disconnect/reconnect. :avocado: tags=all,object,full_regression,small,array_object """ self.d_log.info("Writing the Array Dataset") record_index = 0 transaction = [] for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): c_values = [] value = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) for item in range(self.array_size): c_values.append( (ctypes.create_string_buffer(value), len(value) + 1)) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) new_transaction = self.container.get_new_tx() self.ioreq.insert_array(c_dkey, c_akey, c_values, new_transaction) self.container.commit_tx(new_transaction) transaction.append(new_transaction) record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0 self.reconnect() self.d_log.info("Array Dataset Verification -- Started") record_index = 0 transaction_index = 0 for dkey in range(self.no_of_dkeys): for akey in range(self.no_of_akeys): indata = [] value = ("{0}".format(str(akey)[0]) * self.record_length[record_index]) for item in range(self.array_size): indata.append(value) c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey)) c_akey = ctypes.create_string_buffer("akey {0}".format(akey)) c_rec_count = ctypes.c_uint(len(indata)) c_rec_size = ctypes.c_size_t(len(indata[0]) + 1) outdata = self.ioreq.fetch_array(c_dkey, c_akey, c_rec_count, c_rec_size) for item in enumerate(indata): if indata[item[0]] != outdata[item[0]][:-1]: self.d_log.error("ERROR:Data mismatch for " "dkey = {0}, " "akey = {1}".format( "dkey {0}".format(dkey), "akey {0}".format(akey))) self.fail( "ERROR:Data mismatch for dkey = {0}, akey={1}". format("dkey {0}".format(dkey), "akey {0}".format(akey))) transaction_index = transaction_index + 1 record_index = record_index + 1 if record_index == len(self.record_length): record_index = 0
class PunchTest(TestWithServers): """ Simple test to verify the 3 different punch calls. :avocado: recursive """ def setUp(self): super(PunchTest, self).setUp() self.prepare_pool() try: # create a container self.container = DaosContainer(self.context) self.container.create(self.pool.pool.handle) # now open it self.container.open() except DaosApiError as excpn: print(excpn) print(traceback.format_exc()) self.fail("Test failed during setup.\n") def test_dkey_punch(self): """ The most basic test of the dkey punch function. :avocado: tags=all,object,pr,small,dkeypunch """ try: # create an object and write some data into it thedata = "a string that I want to stuff into an object" dkey = "this is the dkey" akey = "this is the akey" tx_handle = self.container.get_new_tx() print("Created a new TX for punch dkey test") obj = self.container.write_an_obj(thedata, len(thedata)+1, dkey, akey, obj_cls=1, txn=tx_handle) print("Committing the TX for punch dkey test") self.container.commit_tx(tx_handle) print("Committed the TX for punch dkey test") # read the data back and make sure its correct thedata2 = self.container.read_an_obj(len(thedata)+1, dkey, akey, obj, txn=tx_handle) if thedata != thedata2.value: print("data I wrote:" + thedata) print("data I read back" + thedata2.value) self.fail("Wrote data, read it back, didn't match\n") # now punch this data, should fail, can't punch committed data obj.punch_dkeys(tx_handle, [dkey]) # expecting punch of commit data above to fail self.fail("Punch should have failed but it didn't.\n") # expecting an exception so do nothing except DaosApiError as dummy_e: pass try: self.container.close_tx(tx_handle) print("Closed TX for punch dkey test") # now punch this data obj.punch_dkeys(0, [dkey]) # this one should work so error if exception occurs except DaosApiError as dummy_e: self.fail("Punch should have worked.\n") # there are a bunch of other cases to test here, # --test punching the same updating and punching the same data in # the same tx, should fail # --test non updated data in an open tx, should work def test_akey_punch(self): """ The most basic test of the akey punch function. :avocado: tags=all,object,pr,small,akeypunch """ try: # create an object and write some data into it dkey = "this is the dkey" data1 = [("this is akey 1", "this is data value 1"), ("this is akey 2", "this is data value 2"), ("this is akey 3", "this is data value 3")] tx_handle = self.container.get_new_tx() print("Created a new TX for punch akey test") obj = self.container.write_multi_akeys(dkey, data1, obj_cls=1, txn=tx_handle) print("Committing the TX for punch akey test") self.container.commit_tx(tx_handle) print("Committed the TX for punch dkey test") # read back the 1st epoch's data and check 1 value just to make sure # everything is on the up and up readbuf = [(data1[0][0], len(data1[0][1]) + 1), (data1[1][0], len(data1[1][1]) + 1), (data1[2][0], len(data1[2][1]) + 1)] retrieved_data = self.container.read_multi_akeys(dkey, readbuf, obj, txn=tx_handle) if retrieved_data[data1[1][0]] != data1[1][1]: print("middle akey: {}".format(retrieved_data[data1[1][0]])) self.fail("data retrieval failure") # now punch one akey from this data obj.punch_akeys(tx_handle, dkey, [data1[1][0]]) # expecting punch of commit data above to fail self.fail("Punch should have failed but it didn't.\n") # expecting an exception so do nothing except DaosApiError as excep: print(excep) try: self.container.close_tx(tx_handle) print("Closed TX for punch akey test") # now punch the object without a tx obj.punch_akeys(0, dkey, [data1[1][0]]) # expecting it to work this time so error except DaosApiError as excep: self.fail("Punch should have worked: {}\n".format(excep)) def test_obj_punch(self): """ The most basic test of the object punch function. Really similar to above except the whole object is deleted. :avocado: tags=all,object,pr,small,objpunch """ try: # create an object and write some data into it thedata = "a string that I want to stuff into an object" dkey = "this is the dkey" akey = "this is the akey" tx_handle = self.container.get_new_tx() print("Created a new TX for punch obj test") obj = self.container.write_an_obj(thedata, len(thedata)+1, dkey, akey, obj_cls=1, txn=tx_handle) print("Committing the TX for punch obj test") self.container.commit_tx(tx_handle) print("Committed the TX for punch obj test") # read the data back and make sure its correct thedata2 = self.container.read_an_obj(len(thedata)+1, dkey, akey, obj, txn=tx_handle) if thedata != thedata2.value: print("data I wrote:" + thedata) print("data I read back" + thedata2.value) self.fail("Wrote data, read it back, didn't match\n") # now punch the object, commited so not expecting it to work obj.punch(tx_handle) # expecting punch of commit data above to fail self.fail("Punch should have failed but it didn't.\n") # expecting an exception so do nothing except DaosApiError as excep: print(excep) try: self.container.close_tx(tx_handle) print("Closed TX for punch obj test") obj.punch(0) # expecting it to work without a tx except DaosApiError as excep: print(excep) self.fail("Punch should have worked.\n")