class CreateContainerTest(TestWithServers): """ Tests DAOS container create. :avocado: recursive """ def test_container_create(self): """ Test ID: DAOS-689 Test Description: valid and invalid container creation and close. :avocado: tags=all,container,tiny,smoke,full_regression,containercreate """ contuuid = None expected_results = [] # setup the pool self.prepare_pool() # maybe use the good handle, maybe not handleparam = self.params.get("handle", '/run/poolhandle/*') if handleparam == 'VALID': poh = self.pool.pool.handle else: poh = handleparam expected_results.append('FAIL') # maybe use a good UUID, maybe not uuidparam = self.params.get("uuid", "/uuids/*") expected_results.append(uuidparam[1]) if uuidparam[0] == 'NULLPTR': contuuid = 'NULLPTR' else: contuuid = uuid.UUID(uuidparam[0]) should_fail = False for result in expected_results: if result == 'FAIL': should_fail = True break try: self.container = DaosContainer(self.context) self.container.create(poh, contuuid) # check UUID is the specified one if (uuidparam[0]).upper() != self.container.get_uuid_str().upper(): print("uuidparam[0] is {}, uuid_str is {}".format( uuidparam[0], self.container.get_uuid_str())) self.fail("Container UUID differs from specified at create\n") if should_fail: self.fail("Test was expected to fail but it passed.\n") except DaosApiError as excep: print(excep) print(traceback.format_exc()) if not should_fail: self.fail("Test was expected to pass but it failed.\n")
def test_bad_handle(self): """ Test ID: DAOS-1376 Test Description: Pass a bogus object handle, should return bad handle. :avocado: tags=all,object,full_regression,small,objbadhand """ self.prepare_pool() try: # create a container container = DaosContainer(self.context) container.create(self.pool.pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # create an object and write some data into it thedata = "a string that I want to stuff into an object" thedatasize = len(thedata) + 1 dkey = "this is the dkey" akey = "this is the akey" obj = container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) saved_oh = obj.obj_handle obj.obj_handle = 99999 obj = container.write_an_obj(thedata, thedatasize, dkey, akey, obj, None, 2) container.oh = saved_oh container.close() container.destroy() self.fail("Test was expected to return a -1002 but it has not.\n") except DaosApiError as excep: container.oh = saved_oh container.close() container.destroy() self.plog.info("Test Complete") if '-1002' not in str(excep): print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1002 but it has not.\n")
def test_tx_basics(self): """ Perform I/O to an object in a container in 2 different transactions, verifying basic I/O and transactions in particular. NOTE: this was an epoch test and all I did was get it working with tx Not a good test at this point, need to redesign when tx is fully working. :avocado: tags=all,container,tx,small,smoke,pr,basictx """ self.pool = None try: # parameters used in pool create createmode = self.params.get("mode", '/run/poolparams/createmode/') createuid = os.geteuid() creategid = os.getegid() createsetid = self.params.get("setname", '/run/poolparams/createset/') createsize = self.params.get("size", '/run/poolparams/createsize/') # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) # need a connection to create container self.pool.connect(1 << 1) # create a container container = DaosContainer(self.context) container.create(self.pool.handle) # now open it container.open() # do a query and compare the UUID returned from create with # that returned by query container.query() if container.get_uuid_str() != c_uuid_to_str( container.info.ci_uuid): self.fail("Container UUID did not match the one in info\n") # create an object and write some data into it thedata = "a string that I want to stuff into an object" thedatasize = 45 dkey = "this is the dkey" akey = "this is the akey" oid, txn = container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) # read the data back and make sure its correct thedata2 = container.read_an_obj(thedatasize, dkey, akey, oid, txn) if thedata != thedata2.value: print("thedata>" + thedata) print("thedata2>" + thedata2.value) self.fail("Write data 1, read it back, didn't match\n") # repeat above, but know that the write_an_obj call is advancing # the epoch so the original copy remains and the new copy is in # a new epoch. thedata3 = "a different string" thedatasize2 = 19 # note using the same keys so writing to the same spot dkey = "this is the dkey" akey = "this is the akey" oid, tx2 = container.write_an_obj(thedata3, thedatasize2, dkey, akey, oid, None, 2) # read the data back and make sure its correct thedata4 = container.read_an_obj(thedatasize2, dkey, akey, oid, tx2) if thedata3 != thedata4.value: self.fail("Write data 2, read it back, didn't match\n") # transactions generally don't work this way but need to explore # an alternative to below code once model is complete, maybe # read from a snapshot or read from TX_NONE etc. # the original data should still be there too #thedata5 = container.read_an_obj(thedatasize, dkey, akey, # oid, transaction) #if thedata != thedata5.value: # self.fail("Write data 3, read it back, didn't match\n") container.close() # wait a few seconds and then destroy time.sleep(5) container.destroy() except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test was expected to pass but it failed.\n")
def test_array_obj(self): """ Test ID: DAOS-961 Test Description: Writes an array to an object and then reads it back and verifies it. :avocado: tags=all,smoke,pr,object,tiny,basicobject """ self.prepare_pool() try: # create a container container = DaosContainer(self.context) container.create(self.pool.pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # do a query and compare the UUID returned from create with # that returned by query container.query() if container.get_uuid_str() != c_uuid_to_str( container.info.ci_uuid): self.fail("Container UUID did not match the one in info\n") # create an object and write some data into it thedata = [] thedata.append("data string one") thedata.append("data string two") thedata.append("data string tre") dkey = "this is the dkey" akey = "this is the akey" self.plog.info("writing array to dkey >%s< akey >%s<.", dkey, akey) oid = container.write_an_array_value(thedata, dkey, akey, obj_cls=3) # read the data back and make sure its correct length = len(thedata[0]) thedata2 = container.read_an_array(len(thedata), length + 1, dkey, akey, oid) if thedata[0][0:length - 1] != thedata2[0][0:length - 1]: self.plog.error("Data mismatch") self.plog.error("Wrote: >%s<", thedata[0]) self.plog.error("Read: >%s<", thedata2[0]) self.fail("Write data, read it back, didn't match\n") if thedata[2][0:length - 1] != thedata2[2][0:length - 1]: self.plog.error("Data mismatch") self.plog.error("Wrote: >%s<", thedata[2]) self.plog.error("Read: >%s<", thedata2[2]) self.fail("Write data, read it back, didn't match\n") container.close() # wait a few seconds and then destroy time.sleep(5) container.destroy() self.plog.info("Test Complete") except DaosApiError as excep: self.plog.error("Test Failed, exception was thrown.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to pass but it failed.\n")
class Snapshot(TestWithServers): """ Epic: DAOS-2249 Create system level tests that cover basic snapshot functionality. Testcase: DAOS-1370 Basic snapshot test DAOS-1386 Test container SnapShot information DAOS-1371 Test list snapshots DAOS-1395 Test snapshot destroy DAOS-1402 Test creating multiple snapshots Test Class Description: Start DAOS servers, set up the pool and container for the above snapshot Epic and Testcases, including snapshot basic, container information, list, creation and destroy. :avocado: recursive """ def setUp(self): """ set up method """ super(Snapshot, self).setUp() self.log.info("==In setUp, self.context= %s", self.context) # initialize a python pool object then create the underlying # daos storage and connect to it self.prepare_pool() try: # create a container self.container = DaosContainer(self.context) self.container.create(self.pool.pool.handle) except DaosApiError as error: self.log.info("Error detected in DAOS pool container setup: %s", str(error)) self.log.info(traceback.format_exc()) self.fail("##Test failed on setUp, before snapshot taken") # now open it self.container.open() # do a query and compare the UUID returned from create with # that returned by query self.container.query() if self.container.get_uuid_str() != c_uuid_to_str( self.container.info.ci_uuid): self.fail("##Container UUID did not match the one in info.") def display_snapshot(self, snapshot): """ To display the snapshot information. Args: snapshot: snapshot handle to be displayed. Return: none. """ self.log.info("==display_snapshot================") self.log.info("snapshot= %s", snapshot) self.log.info("snapshot.context= %s", snapshot.context) self.log.info("snapshot.context.libdaos= %s", snapshot.context.libdaos) self.log.info("snapshot.context.libtest= %s", snapshot.context.libtest) self.log.info("snapshot.context.ftable= %s", snapshot.context.ftable) self.log.info("snapshot.context.ftable[list-attr]= %s", snapshot.context.ftable["list-attr"]) self.log.info("snapshot.context.ftable[test-event]=%s", snapshot.context.ftable["test-event"]) self.log.info("snapshot.name= %s", snapshot.name) self.log.info("snapshot.epoch= %s", snapshot.epoch) self.log.info("==================================") def take_snapshot(self, container): """ To take a snapshot on the container on current epoch. Args: container: container for the snapshot Return: An object representing the snapshot """ self.log.info("==Taking snapshot for:") self.log.info(" coh= %s", container.coh) snapshot = DaosSnapshot(self.context) snapshot.create(container.coh) self.display_snapshot(snapshot) return snapshot def invalid_snapshot_test(self, coh): """ Negative snapshot test with invalid container handle. Args: container: container for the snapshot Return: 0: Failed 1: Passed (expected failure detected) """ status = 0 try: snapshot = DaosSnapshot(self.context) snapshot.create(coh) except Exception as error: self.log.info("==>Negative test, expected error: %s", str(error)) status = 1 return status def test_snapshot_negativecases(self): # pylint: disable=no-member """ Test ID: DAOS-1390 Verify snap_create bad parameter behavior. DAOS-1322 Create a new container, verify snapshot state. as expected for a brand new container. DAOS-1392 Verify snap_destroy bad parameter behavior. DAOS-1388 Verify snap_list bad parameter behavior. Test Description: (0)Take a snapshot of the newly created container. (1)Create an object, write random data into it, and take a snapshot. (2)Verify the snapshot is working properly. (3)Test snapshot with an invalid container handle. (4)Test snapshot with a NULL container handle. (5)Verify snap_destroy with a bad parameter. (6)Verify snap_list bad parameter behavior. Use Cases: Combinations with minimum 1 client and 1 server. :avocado: tags=all,small,smoke,daily_regression,snap,snapshot_negative, :avocado: tags=snapshotcreate_negative """ #DAOS-1322 Create a new container, verify snapshot state as expected # for a brand new container. try: self.log.info( "==(0)Take a snapshot of the newly created container.") snapshot = DaosSnapshot(self.context) snapshot.create(self.container.coh) self.display_snapshot(snapshot) except Exception as error: self.fail("##(0)Error on a snapshot on a new container %s", str(error)) #(1)Create an object, write some data into it, and take a snapshot obj_cls = self.params.get("obj_class", '/run/object_class/*') akey = self.params.get("akey", '/run/snapshot/*', default="akey") dkey = self.params.get("dkey", '/run/snapshot/*', default="dkey") data_size = self.params.get("test_datasize", '/run/snapshot/*', default=150) rand_str = lambda n: ''.join( [random.choice(string.lowercase) for i in range(n)]) thedata = "--->>>Happy Daos Snapshot-Create Negative Testing " + \ "<<<---" + rand_str(random.randint(1, data_size)) try: obj = self.container.write_an_obj(thedata, len(thedata) + 1, dkey, akey, obj_cls=obj_cls) except DaosApiError as error: self.fail("##(1)Test failed during the initial object write: %s", str(error)) obj.close() ##Take a snapshot of the container snapshot = self.take_snapshot(self.container) self.log.info("==(1)snapshot.epoch= %s", snapshot.epoch) #(2)Verify the snapshot is working properly. try: obj.open() snap_handle = snapshot.open(self.container.coh, snapshot.epoch) thedata2 = self.container.read_an_obj(len(thedata) + 1, dkey, akey, obj, txn=snap_handle.value) except Exception as error: self.fail("##(2)Error when retrieving the snapshot data: %s", str(error)) self.log.info("==(2)snapshot_list[ind]=%s", snapshot) self.log.info("==snapshot.epoch= %s", snapshot.epoch) self.log.info("==written thedata=%s", thedata) self.log.info("==thedata2.value= %s", thedata2.value) if thedata2.value != thedata: raise Exception("##(2)The data in the snapshot is not the " "same as the original data") self.log.info("==Snapshot data matches the data originally " "written.") #(3)Test snapshot with an invalid container handle self.log.info("==(3)Snapshot with an invalid container handle.") if self.invalid_snapshot_test(self.container): self.log.info( "==>Negative test 1, expecting failed on taking " "snapshot with an invalid container.coh: %s", self.container) else: self.fail( "##(3)Negative test 1 passing, expecting failed on" " taking snapshot with an invalid container.coh: %s", self.container) #(4)Test snapshot with a NULL container handle self.log.info("==(4)Snapshot with a NULL container handle.") if self.invalid_snapshot_test(None): self.log.info("==>Negative test 2, expecting failed on taking " "snapshot on a NULL container.coh.") else: self.fail("##(4)Negative test 2 passing, expecting failed on " "taking snapshot with a NULL container.coh.") #(5)DAOS-1392 destroy snapshot with an invalid handle self.log.info( "==(6)DAOS-1392 destroy snapshot with an invalid handle.") try: snapshot.destroy(None, snapshot.epoch) self.fail("##(6)Negative test destroy snapshot with an " "invalid coh handle, expected fail, shown Passing##") except Exception as error: self.log.info( "==>Negative test, destroy snapshot with an invalid handle.") self.log.info(" Expected Error: %s", str(error)) expected_error = "RC: -1002" if expected_error not in str(error): self.fail("##(6.1)Expecting error RC: -1002 did not show.") #(6)DAOS-1388 Verify snap_list bad parameter behavior self.log.info( "==(7)DAOS-1388 Verify snap_list bad parameter behavior.") try: snapshot.list(None, 0) self.fail("##(7)Negative test snapshot list with an " "invalid coh and epoch, expected fail, shown Passing##") except Exception as error: self.log.info( "==>Negative test, snapshot list with an invalid coh.") self.log.info(" Expected Error: %s", str(error)) expected_error = "RC: -1002" if expected_error not in str(error): self.fail("##(7.1)Expecting error RC: -1002 did not show.") def display_snapshot_test_data(self, test_data, ss_index): """Display the snapshot test data. Args: test_data: list of snapshot testdata dictionary keys: coh: container handle snapshot: snapshot handle tst_obj: test object tst_data: test data ss_index: snapshot-list index to be displayed. """ if len(test_data) < ss_index - 1: self.log.info("##Under to display test_data info, " "index out of range.") else: ind = ss_index - 1 self.log.info(" =Snapshot number : %s", ss_index) self.log.info(" ==container_coh =%s", test_data[ind]["coh"]) self.log.info(" ==snapshot =%s", test_data[ind]["snapshot"]) self.log.info(" ==snapshot.epoch =%s", test_data[ind]["snapshot"].epoch) self.log.info(" ==data obj =%s", test_data[ind]["tst_obj"]) self.log.info(" ==snapshot tst_data_size= %s", len(test_data[ind]["tst_data"]) + 1) self.log.info(" ==original tst_data =%s", test_data[ind]["tst_data"]) return def test_snapshots(self): # pylint: disable=no-member,too-many-locals """ Test ID: DAOS-1386 Test container SnapShot information DAOS-1371 Test list snapshots DAOS-1395 Test snapshot destroy DAOS-1402 Test creating multiple snapshots Test Description: (1)Create an object, write random data into it, and take a snapshot. (2)Make changes to the data object. The write_an_obj function does a commit when the update is complete. (3)Verify the data in the snapshot is the original data. Get a handle for the snapshot and read the object at dkey, akey. Compare it to the originally written data. (4)List the snapshot and make sure it reflects the original epoch. ==>Repeat step(1) to step(4) for multiple snapshot tests. (5)Verify the snapshots data. (6)Destroy the snapshot individually. (7)Check if still able to Open the destroyed snapshot and Verify the snapshot removed from the snapshot list. (8)Destroy the container snapshot. Use Cases: Require 1 client and 1 server to run snapshot test. 1 pool and 1 container is used, num_of_snapshot defined in the snapshot.yaml will be performed and verified. :avocado: tags=all,small,smoke,snap,snapshots,full_regression """ test_data = [] ss_number = 0 obj_cls = self.params.get("obj_class", '/run/object_class/*') akey = self.params.get("akey", '/run/snapshot/*', default="akey") dkey = self.params.get("dkey", '/run/snapshot/*', default="dkey") data_size = self.params.get("test_datasize", '/run/snapshot/*', default=150) snapshot_loop = self.params.get("num_of_snapshot", '/run/snapshot/*', default=3) rand_str = lambda n: ''.join( [random.choice(string.lowercase) for i in range(n)]) # #Test loop for creat, modify and snapshot object in the DAOS container. # while ss_number < snapshot_loop: #(1)Create an object, write some data into it, and take a snapshot ss_number += 1 thedata = "--->>>Happy Daos Snapshot Testing " + \ str(ss_number) + \ "<<<---" + rand_str(random.randint(1, data_size)) datasize = len(thedata) + 1 try: obj = self.container.write_an_obj(thedata, datasize, dkey, akey, obj_cls=obj_cls) obj.close() except DaosApiError as error: self.fail("##(1)Test failed during the initial object " "write: {}".format(str(error))) #Take a snapshot of the container snapshot = DaosSnapshot(self.context) snapshot.create(self.container.coh) self.log.info("==Wrote an object and created a snapshot") #Display snapshot self.log.info("=(1.%s)snapshot test loop: %s", ss_number, ss_number) self.log.info(" ==snapshot.epoch= %s", snapshot.epoch) self.display_snapshot(snapshot) #Save snapshot test data test_data.append({ "coh": self.container.coh, "tst_obj": obj, "snapshot": snapshot, "tst_data": thedata }) #(2)Make changes to the data object. The write_an_obj function does # a commit when the update is complete num_transactions = more_transactions = 200 self.log.info( "=(2.%s)Committing %d additional transactions to " "the same KV.", ss_number, more_transactions) while more_transactions: size = random.randint(1, 250) + 1 new_data = rand_str(size) try: new_obj = self.container.write_an_obj(new_data, size, dkey, akey, obj_cls=obj_cls) new_obj.close() except Exception as error: self.fail("##(2)Test failed during the write of " "multi-objects: {}".format(str(error))) more_transactions -= 1 #(3)Verify the data in the snapshot is the original data. # Get a handle for the snapshot and read the object at dkey, akey # Compare it to the originally written data. self.log.info("=(3.%s)snapshot test loop: %s", ss_number, ss_number) try: obj.open() snap_handle = snapshot.open(self.container.coh, snapshot.epoch) thedata3 = self.container.read_an_obj(datasize, dkey, akey, obj, txn=snap_handle.value) obj.close() except Exception as error: self.fail("##(3.1)Error when retrieving the snapshot data: {}". format(str(error))) self.display_snapshot_test_data(test_data, ss_number) self.log.info(" ==thedata3.value= %s", thedata3.value) if thedata3.value != thedata: raise Exception("##(3.2)The data in the snapshot is not the " "same as the original data") self.log.info(" ==The snapshot data matches the data originally" " written.") #(4)List the snapshot and make sure it reflects the original epoch try: ss_list = snapshot.list(self.container.coh, snapshot.epoch) self.log.info("=(4.%s)snapshot.list(self.container.coh)= %s", ss_number, ss_list) self.log.info(" ==snapshot.epoch= %s", snapshot.epoch) except Exception as error: self.fail( "##(4)Test was unable to list the snapshot: {}".format( str(error))) self.log.info( " ==After %s additional commits the snapshot is " "still available", num_transactions) #(5)Verify the snapshots data for ind, _ in enumerate(test_data): ss_number = ind + 1 self.log.info("=(5.%s)Verify the snapshot number %s:", ss_number, ss_number) self.display_snapshot_test_data(test_data, ss_number) coh = test_data[ind]["coh"] current_ss = test_data[ind]["snapshot"] obj = test_data[ind]["tst_obj"] tst_data = test_data[ind]["tst_data"] datasize = len(tst_data) + 1 try: obj.open() snap_handle5 = snapshot.open(coh, current_ss.epoch) thedata5 = self.container.read_an_obj(datasize, dkey, akey, obj, txn=snap_handle5.value) obj.close() except Exception as error: self.fail("##(5.1)Error when retrieving the snapshot data: {}". format(str(error))) self.log.info(" ==snapshot tst_data =%s", thedata5.value) if thedata5.value != tst_data: raise Exception( "##(5.2)Snapshot #%s, test data Mis-matches" "the original data written.", ss_number) self.log.info( " snapshot test number %s, test data matches" " the original data written.", ss_number) #(6)Destroy the individual snapshot self.log.info("=(6.%s)Destroy the snapshot epoch: %s", ss_number, snapshot.epoch) try: snapshot.destroy(coh, snapshot.epoch) self.log.info(" ==snapshot.epoch %s successfully destroyed", snapshot.epoch) except Exception as error: self.fail("##(6)Error on snapshot.destroy: {}".format( str(error))) #(7)Check if still able to Open the destroyed snapshot and # Verify the snapshot removed from the snapshot list try: obj.open() snap_handle7 = snapshot.open(coh, snapshot.epoch) thedata7 = self.container.read_an_obj(datasize, dkey, akey, obj, txn=snap_handle7.value) obj.close() except Exception as error: self.fail( "##(7)Error when retrieving the snapshot data: {}".format( str(error))) self.log.info("=(7)=>thedata_after_snapshot.destroyed.value= %s", thedata7.value) self.log.info(" ==>snapshot.epoch= %s", snapshot.epoch) #Still able to open the snapshot and read data after destroyed. try: ss_list = snapshot.list(coh, snapshot.epoch) self.log.info(" -->snapshot.list(coh, snapshot.epoch)= %s", ss_list) except Exception as error: self.fail("##(7)Error when calling the snapshot list: {}".format( str(error))) #(8)Destroy the snapshot on the container try: snapshot.destroy(coh) self.log.info("=(8)Container snapshot destroyed successfully.") except Exception as error: self.fail("##(8)Error on snapshot.destroy. {}".format(str(error))) self.log.info("===DAOS container Multiple snapshots test passed.")
class TestContainer(TestDaosApiBase): """A class for functional testing of DaosContainer objects.""" def __init__(self, pool, cb_handler=None): """Create a TeestContainer object. Args: pool (TestPool): the test pool in which to create the container cb_handler (CallbackHandler, optional): callback object to use with the API methods. Defaults to None. """ super(TestContainer, self).__init__("/run/container/*", cb_handler) self.pool = pool self.object_qty = BasicParameter(None) self.record_qty = BasicParameter(None) self.akey_size = BasicParameter(None) self.dkey_size = BasicParameter(None) self.data_size = BasicParameter(None) self.data_array_size = BasicParameter(0, 0) self.container = None self.uuid = None self.info = None self.opened = False self.written_data = [] @fail_on(DaosApiError) def create(self, uuid=None): """Create a container. Args: uuid (str, optional): contianer uuid. Defaults to None. """ self.destroy() self.log.info( "Creating a container with pool handle %s", self.pool.pool.handle.value) self.container = DaosContainer(self.pool.context) kwargs = {"poh": self.pool.pool.handle} if uuid is not None: kwargs["con_uuid"] = uuid self._call_method(self.container.create, kwargs) self.uuid = self.container.get_uuid_str() self.log.info(" Container created with uuid %s", self.uuid) @fail_on(DaosApiError) def open(self, pool_handle=None, container_uuid=None): """Open the container with pool handle and container UUID if provided. Args: pool_handle (TestPool.pool.handle, optional): Pool handle. Defaults to None. If you don't provide it, the default pool handle in DaosContainer will be used. If you created a TestPool instance and want to use its pool handle, pass in something like self.pool[-1].pool.handle.value container_uuid (hex, optional): Container UUID. Defaults to None. If you want to use certain container's UUID, pass in something like uuid.UUID(self.container[-1].uuid) Returns: bool: True if the container has been opened; False if the container is already opened. """ if self.container and not self.opened: self.log.info("Opening container %s", self.uuid) kwargs = {} kwargs["poh"] = pool_handle kwargs["cuuid"] = container_uuid self._call_method(self.container.open, kwargs) self.opened = True return True return False @fail_on(DaosApiError) def close(self): """Close the container. Returns: bool: True if the container has been closed; False if the container is already closed. """ if self.container and self.opened: self.log.info("Closing container %s", self.uuid) self._call_method(self.container.close, {}) self.opened = False return True return False @fail_on(DaosApiError) def destroy(self, force=1): """Destroy the container. Args: force (int, optional): force flag. Defaults to 1. Returns: bool: True if the container has been destroyed; False if the container does not exist. """ if self.container: self.close() self.log.info("Destroying container %s", self.uuid) if self.container.attached: self._call_method(self.container.destroy, {"force": force}) self.container = None self.uuid = None self.info = None self.written_data = [] return True return False @fail_on(DaosApiError) def get_info(self, coh=None): """Query the container for information. Sets the self.info attribute. Args: coh (str, optional): container handle override. Defaults to None. """ if self.container: self.open() self.log.info("Querying container %s", self.uuid) self._call_method(self.container.query, {"coh": coh}) self.info = self.container.info def check_container_info(self, ci_uuid=None, es_hce=None, es_lre=None, es_lhe=None, es_ghce=None, es_glre=None, es_ghpce=None, ci_nsnapshots=None, ci_min_slipped_epoch=None): # pylint: disable=unused-argument """Check the container info attributes. Note: Arguments may also be provided as a string with a number preceeded by '<', '<=', '>', or '>=' for other comparisions besides the default '=='. Args: ci_uuid (str, optional): container uuid. Defaults to None. es_hce (int, optional): hc epoch?. Defaults to None. es_lre (int, optional): lr epoch?. Defaults to None. es_lhe (int, optional): lh epoch?. Defaults to None. es_ghce (int, optional): ghc epoch?. Defaults to None. es_glre (int, optional): glr epoch?. Defaults to None. es_ghpce (int, optional): ghpc epoch?. Defaults to None. ci_nsnapshots (int, optional): number of snapshots. Defaults to None. ci_min_slipped_epoch (int, optional): . Defaults to None. Note: Arguments may also be provided as a string with a number preceeded by '<', '<=', '>', or '>=' for other comparisions besides the default '=='. Returns: bool: True if at least one expected value is specified and all the specified values match; False otherwise """ self.get_info() checks = [ (key, c_uuid_to_str(getattr(self.info, key)) if key == "ci_uuid" else getattr(self.info, key), val) for key, val in locals().items() if key != "self" and val is not None] return self._check_info(checks) @fail_on(DaosTestError) def write_objects(self, rank=None, obj_class=None, debug=False): """Write objects to the container. Args: rank (int, optional): server rank. Defaults to None. obj_class (int, optional): daos object class. Defaults to None. debug (bool, optional): log the record write/read method calls. Defaults to False. Raises: DaosTestError: if there was an error writing the object """ self.open() self.log.info( "Writing %s object(s), with %s record(s) of %s bytes(s) each, in " "container %s%s%s", self.object_qty.value, self.record_qty.value, self.data_size.value, self.uuid, " on rank {}".format(rank) if rank is not None else "", " with object class {}".format(obj_class) if obj_class is not None else "") for _ in range(self.object_qty.value): self.written_data.append(TestContainerData(debug)) self.written_data[-1].write_object( self, self.record_qty.value, self.akey_size.value, self.dkey_size.value, self.data_size.value, rank, obj_class, self.data_array_size.value) @fail_on(DaosTestError) def read_objects(self, debug=False): """Read the objects from the container and verify they match. Args: debug (bool, optional): log the record read method calls. Defaults to False. Returns: bool: True if """ self.open() self.log.info( "Reading %s object(s) in container %s", len(self.written_data), self.uuid) status = len(self.written_data) > 0 for data in self.written_data: data.debug = debug status &= data.read_object(self) return status def execute_io(self, duration, rank=None, obj_class=None, debug=False): """Execute writes and reads for the specified time period. Args: duration (int): how long, in seconds, to write and read data rank (int, optional): server rank. Defaults to None. obj_class (int, optional): daos object class. Defaults to None. debug (bool, optional): log the record write/read method calls. Defaults to False. Returns: int: number of bytes written to the container Raises: DaosTestError: if there is an error writing, reading, or verify the data """ self.open() self.log.info( "Writing and reading objects in container %s for %s seconds", self.uuid, duration) total_bytes_written = 0 finish_time = time() + duration while time() < finish_time: self.written_data.append(TestContainerData(debug)) self.written_data[-1].write_object( self, 1, self.akey_size.value, self.dkey_size.value, self.data_size.value, rank, obj_class) total_bytes_written += self.data_size.value return total_bytes_written def get_target_rank_lists(self, message=""): """Get a list of lists of target ranks from each written object. Args: message (str, optional): message to include in the target rank list output. Defaults to "". Raises: DaosTestError: if there is an error obtaining the target rank list from the DaosObj Returns: list: a list of list of targets for each written object in this container """ self.open() target_rank_lists = [] for data in self.written_data: try: data.obj.get_layout() # Convert the list of longs into a list of ints target_rank_lists.append( [int(rank) for rank in data.obj.tgt_rank_list]) except DaosApiError as error: raise DaosTestError( "Error obtaining target rank list for object {} in " "container {}: {}".format(data.obj, self.uuid, error)) if message is not None: self.log.info("Target rank lists%s:", message) for ranks in target_rank_lists: self.log.info(" %s", ranks) return target_rank_lists def get_target_rank_count(self, rank, target_rank_list): """Get the number of times a rank appears in the target rank list. Args: rank (int): the rank to count. Defaults to None. target_rank_list (list): a list of lists of target ranks per object Returns: (int): the number of object rank lists containing the rank """ count = sum([ranks.count(rank) for ranks in target_rank_list]) self.log.info( "Occurrences of rank %s in the target rank list: %s", rank, count) return count def punch_objects(self, indices): """Punch committed objects from the container. Args: indices (list): list of index numbers indicating which written objects to punch from the self.written_data list Raises: DaosTestError: if there is an error punching the object or determining which objec to punch Returns: int: number of successfully punched objects """ self.open() self.log.info( "Punching %s objects from container %s with %s written objects", len(indices), self.uuid, len(self.written_data)) count = 0 if len(self.written_data) > 0: for index in indices: # Find the object to punch at the specified index txn = 0 try: obj = self.written_data[index].obj except IndexError: raise DaosTestError( "Invalid index {} for written data".format(index)) # Close the object self.log.info( "Closing object (index: %s, txn: %s) in container %s", index, txn, self.uuid) try: self._call_method(obj.close, {}) except DaosApiError as error: self.log.error(" Error: %s", str(error)) continue # Punch the object self.log.info( "Punching object (index: %s, txn: %s) from container %s", index, txn, self.uuid) try: self._call_method(obj.punch, {"txn": txn}) count += 1 except DaosApiError as error: self.log.error(" Error: %s", str(error)) # Retutrn the number of punched objects return count
def test_array_obj(self): """ Test ID: DAOS-961 Test Description: Writes an array to an object and then reads it back and verifies it. :avocado: tags=all,smoke,pr,object,tiny,basicobject """ try: # parameters used in pool create createmode = self.params.get("mode", '/run/pool_params/createmode/') createsetid = self.params.get("setname", '/run/pool_params/createset/') createsize = self.params.get("size", '/run/pool_params/createsize/') createuid = os.geteuid() creategid = os.getegid() # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) self.plog.info("Pool %s created.", pool.get_uuid_str()) # need a connection to create container pool.connect(1 << 1) # create a container container = DaosContainer(self.context) container.create(pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # do a query and compare the UUID returned from create with # that returned by query container.query() if container.get_uuid_str() != c_uuid_to_str( container.info.ci_uuid): self.fail("Container UUID did not match the one in info\n") # create an object and write some data into it thedata = [] thedata.append("data string one") thedata.append("data string two") thedata.append("data string tre") dkey = "this is the dkey" akey = "this is the akey" self.plog.info("writing array to dkey >%s< akey >%s<.", dkey, akey) oid, epoch = container.write_an_array_value(thedata, dkey, akey, obj_cls=3) # read the data back and make sure its correct length = len(thedata[0]) thedata2 = container.read_an_array(len(thedata), length + 1, dkey, akey, oid, epoch) if thedata[0][0:length - 1] != thedata2[0][0:length - 1]: self.plog.error("Data mismatch") self.plog.error("Wrote: >%s<", thedata[0]) self.plog.error("Read: >%s<", thedata2[0]) self.fail("Write data, read it back, didn't match\n") if thedata[2][0:length - 1] != thedata2[2][0:length - 1]: self.plog.error("Data mismatch") self.plog.error("Wrote: >%s<", thedata[2]) self.plog.error("Read: >%s<", thedata2[2]) self.fail("Write data, read it back, didn't match\n") container.close() # wait a few seconds and then destroy time.sleep(5) container.destroy() # cleanup the pool pool.disconnect() pool.destroy(1) self.plog.info("Test Complete") except DaosApiError as excep: self.plog.error("Test Failed, exception was thrown.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to pass but it failed.\n")
def test_null_values(self): """ Test ID: DAOS-1376 Test Description: Pass a dkey and an akey that is null. :avocado: tags=all,object,full_regression,small,objupdatenull """ self.prepare_pool() try: # create a container container = DaosContainer(self.context) container.create(self.pool.pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # data used in the test thedata = "a string that I want to stuff into an object" thedatasize = len(thedata) + 1 except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test failed during setup .\n") try: # try using a null dkey dkey = None akey = "this is the akey" container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) container.close() container.destroy() self.plog.error("Didn't get expected return code.") self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): container.close() container.destroy() self.plog.error("Didn't get expected return code.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # try using a null akey/io descriptor dkey = "this is the dkey" akey = None container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): self.plog.error("Didn't get expected return code.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # lastly try passing no data thedata = None thedatasize = 0 dkey = "this is the dkey" akey = "this is the akey" container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.plog.info("Update with no data worked") except DaosApiError as excep: container.close() container.destroy() print(excep) print(traceback.format_exc()) self.plog.error("Update with no data failed") self.fail("Update with no data failed.\n") container.close() container.destroy() self.plog.info("Test Complete")
class CreateContainerTest(TestWithServers): """ Tests DAOS container create. :avocado: recursive """ def test_container_create(self): """ Test ID: DAOS-689 Test Description: valid and invalid container creation and close. :avocado: tags=all,container,tiny,smoke,full_regression,containercreate """ pool = None contuuid = None expected_results = [] try: # initialize a python pool object then create the underlying # daos storage createmode = self.params.get("mode", '/run/poolparams/') createuid = os.geteuid() creategid = os.getegid() createsetid = self.params.get("setname", '/run/poolparams/') createsize = self.params.get("size", '/run/poolparams/') # setup the pool pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid) pool.connect(1 << 1) # maybe use the good handle, maybe not handleparam = self.params.get("handle", '/run/poolhandle/*') if handleparam == 'VALID': poh = pool.handle else: poh = handleparam expected_results.append('FAIL') # maybe use a good UUID, maybe not uuidparam = self.params.get("uuid", "/uuids/*") expected_results.append(uuidparam[1]) if uuidparam[0] == 'NULLPTR': self.cancel("skipping this test until DAOS-2043 is fixed") # Commenting the line below as it will result in an # AttributeError and never get to the DAOS API code. # Should be further investigated as part of DAOS-3081 # contuuid = 'NULLPTR' else: contuuid = uuid.UUID(uuidparam[0]) should_fail = False for result in expected_results: if result == 'FAIL': should_fail = True break self.container = DaosContainer(self.context) self.container.create(poh, contuuid) # check UUID is the specified one if (uuidparam[0]).upper() != self.container.get_uuid_str().upper(): print("uuidparam[0] is {}, uuid_str is {}".format( uuidparam[0], self.container.get_uuid_str())) self.fail("Container UUID differs from specified at create\n") if should_fail: self.fail("Test was expected to fail but it passed.\n") except DaosApiError as excep: print(excep) print(traceback.format_exc()) if not should_fail: self.fail("Test was expected to pass but it failed.\n")
def test_bad_handle(self): """ Test ID: DAOS-1376 Test Description: Pass a bogus object handle, should return bad handle. :avocado: tags=all,object,full_regression,small,objbadhand """ try: # parameters used in pool create createmode = self.params.get("mode", '/run/conttests/createmode/') createsetid = self.params.get("setname", '/run/conttests/createset/') createsize = self.params.get("size", '/run/conttests/createsize/') createuid = os.geteuid() creategid = os.getegid() # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) self.plog.info("Pool %s created.", pool.get_uuid_str()) # need a connection to create container pool.connect(1 << 1) # create a container container = DaosContainer(self.context) container.create(pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # create an object and write some data into it thedata = "a string that I want to stuff into an object" thedatasize = len(thedata) + 1 dkey = "this is the dkey" akey = "this is the akey" obj, dummy_tx = container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) saved_oh = obj.obj_handle obj.obj_handle = 99999 obj, dummy_tx = container.write_an_obj(thedata, thedatasize, dkey, akey, obj, None, 2) container.oh = saved_oh container.close() container.destroy() pool.disconnect() pool.destroy(1) self.fail("Test was expected to return a -1002 but it has not.\n") except DaosApiError as excep: container.oh = saved_oh container.close() container.destroy() pool.disconnect() pool.destroy(1) self.plog.info("Test Complete") if '-1002' not in str(excep): print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1002 but it has not.\n")
def test_null_values(self): """ Test ID: DAOS-1376 Test Description: Pass a dkey and an akey that is null. :avocado: tags=all,object,full_regression,small,objupdatenull """ try: # parameters used in pool create createmode = self.params.get("mode", '/run/conttests/createmode/') createsetid = self.params.get("setname", '/run/conttests/createset/') createsize = self.params.get("size", '/run/conttests/createsize/') createuid = os.geteuid() creategid = os.getegid() # initialize a python pool object then create the underlying # daos storage pool = DaosPool(self.context) pool.create(createmode, createuid, creategid, createsize, createsetid, None) self.plog.info("Pool %s created.", pool.get_uuid_str()) # need a connection to create container pool.connect(1 << 1) # create a container container = DaosContainer(self.context) container.create(pool.handle) self.plog.info("Container %s created.", container.get_uuid_str()) # now open it container.open() # data used in the test thedata = "a string that I want to stuff into an object" thedatasize = len(thedata) + 1 except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test failed during setup .\n") try: # try using a null dkey dkey = None akey = "this is the akey" container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) container.close() container.destroy() pool.disconnect() pool.destroy(1) self.plog.error("Didn't get expected return code.") self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): container.close() container.destroy() pool.disconnect() pool.destroy(1) self.plog.error("Didn't get expected return code.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # try using a null akey/io descriptor dkey = "this is the dkey" akey = None container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): self.plog.error("Didn't get expected return code.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # lastly try passing no data thedata = None thedatasize = 0 dkey = "this is the dkey" akey = "this is the akey" container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.plog.info("Update with no data worked") except DaosApiError as excep: container.close() container.destroy() pool.disconnect() pool.destroy(1) print(excep) print(traceback.format_exc()) self.plog.error("Update with no data failed") self.fail("Update with no data failed.\n") container.close() container.destroy() pool.disconnect() pool.destroy(1) self.plog.info("Test Complete")
class ContainerAsync(TestWithServers): """ Tests DAOS pool connect permissions (non existing pool handle, bad uuid) and close. :avocado: recursive """ def __init__(self, *args, **kwargs): super(ContainerAsync, self).__init__(*args, **kwargs) self.container1 = None self.container2 = None self.pool = None def test_createasync(self): """ Test container create for asynchronous mode. :avocado: tags=all,small,full_regression,container,createasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) GLOB_SIGNAL = threading.Event() self.container1.create(poh, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test") print("RC after successful container create: ", GLOB_RC) # Try to recreate container after destroying pool, # this should fail. Checking rc after failure. self.pool.disconnect() self.pool.destroy(1) GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.create(poh, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC == 0: self.fail("RC not as expected in async test") print("RC after unsuccessful container create: ", GLOB_RC) # cleanup the pool and container self.pool = None except DaosApiError as excep: print(excep) print(traceback.format_exc()) def test_destroyasync(self): """ Test container destroy for asynchronous mode. :avocado: tags=all,small,full_regression,container,contdestroyasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) self.container1.create(poh) GLOB_SIGNAL = threading.Event() self.container1.destroy(1, poh, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test") print("RC after successful container create: ", GLOB_RC) # Try to destroy container again, this should fail, as non-existent. # Checking rc after failure. GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.destroy(1, poh, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != -1003: self.fail("RC not as expected in async test") print("RC after container destroy failed:", GLOB_RC) except DaosApiError as excep: print(excep) print(traceback.format_exc()) def test_openasync(self): """ Test container open for asynchronous mode. :avocado: tags=all,small,full_regression,container,openasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) self.container1.create(poh) str_cuuid = self.container1.get_uuid_str() cuuid = uuid.UUID(str_cuuid) GLOB_SIGNAL = threading.Event() self.container1.open(poh, cuuid, 2, cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test") print("RC after successful container create: ", GLOB_RC) # Try to open container2, this should fail, as non-existent. # Checking rc after failure. GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.open(None, None, None, cb_func) GLOB_SIGNAL.wait() if GLOB_RC == 0: self.fail("RC not as expected in async test") print("RC after container destroy failed:", GLOB_RC) # cleanup the container self.container1.close() self.container1.destroy() except DaosApiError as excep: print(excep) print(traceback.format_exc()) def test_closeasync(self): """ Test container close for asynchronous mode. :avocado: tags=all,small,full_regression,container,closeasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) self.container1.create(poh) str_cuuid = self.container1.get_uuid_str() cuuid = uuid.UUID(str_cuuid) self.container1.open(poh, cuuid, 2) GLOB_SIGNAL = threading.Event() self.container1.close(cb_func=cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test: " "{0}".format(GLOB_RC)) print("RC after successful container create: ", GLOB_RC) # Try to open container2, this should fail, as non-existent. # Checking rc after failure. GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.close(cb_func=cb_func) GLOB_SIGNAL.wait() if GLOB_RC == 0: self.fail("RC not as expected in async test: " "{0}".format(GLOB_RC)) print("RC after container destroy failed:", GLOB_RC) # cleanup the container self.container1.destroy() except DaosApiError as excep: print(excep) print(traceback.format_exc()) def test_queryasync(self): """ Test container query for asynchronous mode. :avocado: tags=all,small,full_regression,container,queryasync """ global GLOB_SIGNAL global GLOB_RC # parameters used in pool create createmode = self.params.get("mode", '/run/createtests/createmode/*/') createsetid = self.params.get("setname", '/run/createtests/createset/') createsize = self.params.get("size", '/run/createtests/createsize/') createuid = os.geteuid() creategid = os.getegid() try: # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createsize, createsetid, None) poh = self.pool.handle self.pool.connect(1 << 1) # Container initialization and creation self.container1 = DaosContainer(self.context) self.container2 = DaosContainer(self.context) self.container1.create(poh) dummy_str_cuuid = self.container1.get_uuid_str() # Open container self.container1.open(poh, None, 2, None) GLOB_SIGNAL = threading.Event() self.container1.query(cb_func=cb_func) GLOB_SIGNAL.wait() if GLOB_RC != 0: self.fail("RC not as expected in async test: " "{0}".format(GLOB_RC)) print("RC after successful container create: ", GLOB_RC) # Close opened container self.container1.close() # Try to open container2, this should fail, as non-existent. # Checking rc after failure. GLOB_SIGNAL = threading.Event() GLOB_RC = -9900000 self.container2.query(cb_func=cb_func) GLOB_SIGNAL.wait() if GLOB_RC == 0: self.fail("RC not as expected in async test: " "{0}".format(GLOB_RC)) print("RC after container destroy failed:", GLOB_RC) # cleanup the container self.container1.destroy() except DaosApiError as excep: print(excep) print(traceback.format_exc())
class TestContainer(TestDaosApiBase): """A class for functional testing of DaosContainer objects.""" def __init__(self, pool, cb_handler=None, daos_command=None): """Create a TeestContainer object. Args: pool (TestPool): the test pool in which to create the container cb_handler (CallbackHandler, optional): callback object to use with the API methods. Defaults to None. """ super(TestContainer, self).__init__("/run/container/*", cb_handler) self.pool = pool self.object_qty = BasicParameter(None) self.record_qty = BasicParameter(None) self.akey_size = BasicParameter(None) self.dkey_size = BasicParameter(None) self.data_size = BasicParameter(None) self.data_array_size = BasicParameter(0, 0) # Provider access to get input params values # for enabling different container properties self.input_params = DaosInputParams() # Optional daos command object to use with the USE_DAOS control method self.daos = daos_command # Optional daos command argument values to use with the USE_DAOS control # method when creating/destroying containers self.path = BasicParameter(None) self.type = BasicParameter(None) self.oclass = BasicParameter(None) self.chunk_size = BasicParameter(None) self.properties = BasicParameter(None) self.daos_timeout = BasicParameter(None) self.container = None self.uuid = None self.info = None self.opened = False self.written_data = [] self.epoch = None def __str__(self): """Return a string representation of this TestContainer object. Returns: str: the container's UUID, if defined """ if self.container is not None and self.uuid is not None: return str(self.uuid) return super(TestContainer, self).__str__() def get_params(self, test): """Get values for all of the command params from the yaml file. Sets each BasicParameter object's value to the yaml key that matches the assigned name of the BasicParameter object in this class. For example, the self.block_size.value will be set to the value in the yaml file with the key 'block_size'. If no key matches are found in the yaml file the BasicParameter object will be set to its default value. Args: test (Test): avocado Test object """ super(TestContainer, self).get_params(test) if self.daos: self.daos.timeout = self.daos_timeout.value @fail_on(DaosApiError) @fail_on(CommandFailure) def create(self, uuid=None, con_in=None, acl_file=None): """Create a container. Args: uuid (str, optional): container uuid. Defaults to None. con_in (optional): to be defined. Defaults to None. acl_file (str, optional): path of the ACL file. Defaults to None. """ self.destroy() self.log.info("Creating a container with pool handle %s", self.pool.pool.handle.value) self.container = DaosContainer(self.pool.context) if self.control_method.value == self.USE_API: # Create a container with the API method kwargs = {"poh": self.pool.pool.handle} if uuid is not None: kwargs["con_uuid"] = uuid # Refer daos_api for setting input params for DaosContainer. if con_in is not None: cop = self.input_params.get_con_create_params() cop.type = con_in[0] cop.enable_chksum = con_in[1] cop.srv_verify = con_in[2] cop.chksum_type = con_in[3] cop.chunk_size = con_in[4] kwargs["con_prop"] = cop self._call_method(self.container.create, kwargs) elif self.control_method.value == self.USE_DAOS and self.daos: # Disconnect the pool if connected self.pool.disconnect() # Create a container with the daos command kwargs = { "pool": self.pool.uuid, "sys_name": self.pool.name.value, "cont": uuid, "path": self.path.value, "cont_type": self.type.value, "oclass": self.oclass.value, "chunk_size": self.chunk_size.value, "properties": self.properties.value, "acl_file": acl_file, } self._log_method("daos.container_create", kwargs) uuid = self.daos.get_output("container_create", **kwargs)[0] # Populte the empty DaosContainer object with the properties of the # container created with daos container create. self.container.uuid = str_to_c_uuid(uuid) self.container.attached = 1 self.container.poh = self.pool.pool.handle elif self.control_method.value == self.USE_DAOS: self.log.error("Error: Undefined daos command") else: self.log.error("Error: Undefined control_method: %s", self.control_method.value) self.uuid = self.container.get_uuid_str() self.log.info(" Container created with uuid %s", self.uuid) @fail_on(DaosApiError) @fail_on(CommandFailure) def create_snap(self, snap_name=None, epoch=None): """Create Snapshot using daos utility. Args: snap_name (str, optional): Snapshot name. Defaults to None. epoch (str, optional): Epoch ID. Defaults to None. """ self.log.info("Creating Snapshot for Container: %s", self.uuid) if self.control_method.value == self.USE_DAOS and self.daos: # create snapshot using daos utility kwargs = { "pool": self.pool.uuid, "cont": self.uuid, "snap_name": snap_name, "epoch": epoch, "sys_name": self.pool.name.value } self._log_method("daos.container_create_snap", kwargs) data = self.daos.container_create_snap(**kwargs) elif self.control_method.value == self.USE_DAOS: self.log.error("Error: Undefined daos command") else: self.log.error("Error: Undefined control_method: %s", self.control_method.value) self.epoch = data["epoch"] @fail_on(DaosApiError) @fail_on(CommandFailure) def destroy_snap(self, snap_name=None, epc=None, epcrange=None): """Destroy Snapshot using daos utility. Args: snap_name (str, optional): Snapshot name epc (str, optional): Epoch ID that indicates the snapshot to be destroyed. Defaults to None. epcrange (str, optional): Epoch range in the format "<start>-<end>". """ status = False self.log.info("Destroying Snapshot for Container: %s", self.uuid) if self.control_method.value == self.USE_DAOS and self.daos: # destroy snapshot using daos utility kwargs = { "pool": self.pool.uuid, "cont": self.uuid, "snap_name": snap_name, "epc": epc, "epcrange": epcrange, "sys_name": self.pool.name.value, } self._log_method("daos.container_destroy_snap", kwargs) self.daos.container_destroy_snap(**kwargs) status = True elif self.control_method.value == self.USE_DAOS: self.log.error("Error: Undefined daos command") else: self.log.error("Error: Undefined control_method: %s", self.control_method.value) self.epoch = None return status @fail_on(DaosApiError) def open(self, pool_handle=None, container_uuid=None): """Open the container with pool handle and container UUID if provided. Args: pool_handle (TestPool.pool.handle, optional): Pool handle. Defaults to None. If you don't provide it, the default pool handle in DaosContainer will be used. If you created a TestPool instance and want to use its pool handle, pass in something like self.pool[-1].pool.handle.value container_uuid (hex, optional): Container UUID. Defaults to None. If you want to use certain container's UUID, pass in something like uuid.UUID(self.container[-1].uuid) Returns: bool: True if the container has been opened; False if the container is already opened. """ if self.container and not self.opened: self.log.info("Opening container %s", self.uuid) self.pool.connect() kwargs = {} kwargs["poh"] = pool_handle kwargs["cuuid"] = container_uuid self._call_method(self.container.open, kwargs) self.opened = True return True return False @fail_on(DaosApiError) def close(self): """Close the container. Returns: bool: True if the container has been closed; False if the container is already closed. """ if self.container and self.opened: self.log.info("Closing container %s", self.uuid) self._call_method(self.container.close, {}) self.opened = False return True return False @fail_on(DaosApiError) @fail_on(CommandFailure) def destroy(self, force=1): """Destroy the container. Args: force (int, optional): force flag. Defaults to 1. Returns: bool: True if the container has been destroyed; False if the container does not exist. """ status = False if self.container: self.close() self.log.info("Destroying container %s", self.uuid) if self.container.attached: kwargs = {"force": force} if self.control_method.value == self.USE_API: # Destroy the container with the API method self._call_method(self.container.destroy, kwargs) status = True elif self.control_method.value == self.USE_DAOS and self.daos: # Disconnect the pool if connected self.pool.disconnect() # Destroy the container with the daos command kwargs["pool"] = self.pool.uuid kwargs["sys_name"] = self.pool.name.value kwargs["cont"] = self.uuid self._log_method("daos.container_destroy", kwargs) self.daos.container_destroy(**kwargs) status = True elif self.control_method.value == self.USE_DAOS: self.log.error("Error: Undefined daos command") else: self.log.error("Error: Undefined control_method: %s", self.control_method.value) self.container = None self.uuid = None self.info = None self.written_data = [] return status @fail_on(DaosApiError) def get_info(self, coh=None): """Query the container for information. Sets the self.info attribute. Args: coh (str, optional): container handle override. Defaults to None. """ if self.container: self.open() self.log.info("Querying container %s", self.uuid) self._call_method(self.container.query, {"coh": coh}) self.info = self.container.info def check_container_info(self, ci_uuid=None, ci_nsnapshots=None): # pylint: disable=unused-argument """Check the container info attributes. Note: Arguments may also be provided as a string with a number preceded by '<', '<=', '>', or '>=' for other comparisons besides the default '=='. Args: ci_uuid (str, optional): container uuid. Defaults to None. ci_nsnapshots (int, optional): number of snapshots. Defaults to None. Note: Arguments may also be provided as a string with a number preceded by '<', '<=', '>', or '>=' for other comparisons besides the default '=='. Returns: bool: True if at least one expected value is specified and all the specified values match; False otherwise """ self.get_info() checks = [(key, c_uuid_to_str(getattr(self.info, key)) if key == "ci_uuid" else getattr(self.info, key), val) for key, val in locals().items() if key != "self" and val is not None] return self._check_info(checks) @fail_on(DaosTestError) def write_objects(self, rank=None, obj_class=None): """Write objects to the container. Args: rank (int, optional): server rank. Defaults to None. obj_class (int, optional): daos object class. Defaults to None. Raises: DaosTestError: if there was an error writing the object """ self.open() self.log.info( "Writing %s object(s), with %s record(s) of %s bytes(s) each, in " "container %s%s%s", self.object_qty.value, self.record_qty.value, self.data_size.value, self.uuid, " on rank {}".format(rank) if rank is not None else "", " with object class {}".format(obj_class) if obj_class is not None else "") for _ in range(self.object_qty.value): self.written_data.append(TestContainerData(self.debug.value)) self.written_data[-1].write_object(self, self.record_qty.value, self.akey_size.value, self.dkey_size.value, self.data_size.value, rank, obj_class, self.data_array_size.value) @fail_on(DaosTestError) def read_objects(self, txn=None): """Read the objects from the container and verify they match. Args: txn (int, optional): transaction timestamp to read. Defaults to None which uses the last timestamp written. Returns: bool: True if all the container objects contain the same previously written records; False otherwise """ self.open() self.log.info("Reading %s object(s) in container %s", len(self.written_data), self.uuid) status = len(self.written_data) > 0 for data in self.written_data: data.debug = self.debug.value status &= data.read_object(self, txn) return status def execute_io(self, duration, rank=None, obj_class=None): """Execute writes and reads for the specified time period. Args: duration (int): how long, in seconds, to write and read data rank (int, optional): server rank. Defaults to None. obj_class (int, optional): daos object class. Defaults to None. Returns: int: number of bytes written to the container Raises: DaosTestError: if there is an error writing, reading, or verify the data """ self.open() self.log.info( "Writing and reading objects in container %s for %s seconds", self.uuid, duration) total_bytes_written = 0 finish_time = time() + duration while time() < finish_time: self.written_data.append(TestContainerData(self.debug.value)) self.written_data[-1].write_object(self, 1, self.akey_size.value, self.dkey_size.value, self.data_size.value, rank, obj_class) total_bytes_written += self.data_size.value return total_bytes_written def get_target_rank_lists(self, message=""): """Get a list of lists of target ranks from each written object. Args: message (str, optional): message to include in the target rank list output. Defaults to "". Raises: DaosTestError: if there is an error obtaining the target rank list from the DaosObj Returns: list: a list of list of targets for each written object in this container """ self.open() target_rank_lists = [] for data in self.written_data: try: data.obj.get_layout() # Convert the list of longs into a list of ints target_rank_lists.append( [int(rank) for rank in data.obj.tgt_rank_list]) except DaosApiError as error: raise DaosTestError( "Error obtaining target rank list for object {} in " "container {}: {}".format(data.obj, self.uuid, error)) if message is not None: self.log.info("Target rank lists%s:", message) for ranks in target_rank_lists: self.log.info(" %s", ranks) return target_rank_lists def get_target_rank_count(self, rank, target_rank_list): """Get the number of times a rank appears in the target rank list. Args: rank (int): the rank to count. Defaults to None. target_rank_list (list): a list of lists of target ranks per object Returns: (int): the number of object rank lists containing the rank """ count = sum([ranks.count(rank) for ranks in target_rank_list]) self.log.info("Occurrences of rank %s in the target rank list: %s", rank, count) return count def punch_objects(self, indices): """Punch committed objects from the container. Args: indices (list): list of index numbers indicating which written objects to punch from the self.written_data list Raises: DaosTestError: if there is an error punching the object or determining which object to punch Returns: int: number of successfully punched objects """ self.open() self.log.info( "Punching %s objects from container %s with %s written objects", len(indices), self.uuid, len(self.written_data)) count = 0 if self.written_data: for index in indices: # Find the object to punch at the specified index txn = 0 try: obj = self.written_data[index].obj except IndexError: raise DaosTestError( "Invalid index {} for written data".format(index)) # Close the object self.log.info( "Closing object %s (index: %s, txn: %s) in container %s", obj, index, txn, self.uuid) try: self._call_method(obj.close, {}) except DaosApiError: continue # Punch the object self.log.info( "Punching object %s (index: %s, txn: %s) from container %s", obj, index, txn, self.uuid) try: self._call_method(obj.punch, {"txn": txn}) count += 1 except DaosApiError: continue # Mark the object's records as punched for record in self.written_data[index].records: record["punched"] = True # Retutrn the number of punched objects return count def punch_records(self, indices, punch_dkey=True): """Punch committed records from the container. Args: indices (list): list of index numbers indicating which written records in the object to punch from the self.written_data list punch_dkey (bool, optional): whether to punch dkeys or akeys. Defaults to True (punch dkeys). Raises: DaosTestError: if there is an error punching the record or determining which record to punch Returns: int: number of successfully punched records """ self.open() self.log.info( "Punching %s records from each object in container %s with %s " "written objects", len(indices), self.uuid, len(self.written_data)) count = 0 for data in self.written_data: # Close the object self.log.info("Closing object %s in container %s", data.obj, self.uuid) try: self._call_method(data.obj.close, {}) except DaosApiError: continue # Find the record to punch at the specified index for index in indices: try: rec = data.records[index] except IndexError: raise DaosTestError( "Invalid record index {} for object {}".format( index, data.obj)) # Punch the record self.log.info( "Punching record %s (index: %s, akey: %s, dkey: %s) from " "object %s in container %s", rec, index, rec["akey"], rec["dkey"], data.obj, self.uuid) kwargs = {"txn": 0} try: if punch_dkey: kwargs["dkeys"] = [rec["dkey"]] self._call_method(data.obj.punch_dkeys, kwargs) else: kwargs["dkey"] = rec["dkey"] kwargs["akeys"] = [rec["akey"]] self._call_method(data.obj.punch_akeys, kwargs) count += 1 except DaosApiError: continue # Mark the record as punched rec["punched"] = True # Return the number of punched objects return count
class ObjUpdateBadParam(TestWithServers): """ Test Class Description: Pass an assortment of bad parameters to the daos_obj_update function. :avocado: recursive """ def setUp(self): super().setUp() self.plog = logging.getLogger("progress") try: self.prepare_pool() # create a container self.container = DaosContainer(self.context) self.container.create(self.pool.pool.handle) self.plog.info("Container %s created.", self.container.get_uuid_str()) # now open it self.container.open() except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.fail("Test failed during setup .\n") def test_bad_handle(self): """ Test ID: DAOS-1376 Test Description: Pass a bogus object handle, should return bad handle. :avocado: tags=all,full_regression,small :avocado: tags=object,objupdatebadparam :avocado: tags=objbadhand """ try: # create an object and write some data into it thedata = b"a string that I want to stuff into an object" thedatasize = len(thedata) + 1 dkey = b"this is the dkey" akey = b"this is the akey" obj = self.container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) saved_oh = obj.obj_handle obj.obj_handle = 99999 obj = self.container.write_an_obj(thedata, thedatasize, dkey, akey, obj, None, 2) self.container.oh = saved_oh self.fail("Test was expected to return a -1002 but it has not.\n") except DaosApiError as excep: self.container.oh = saved_oh self.plog.info("Test Complete") if '-1002' not in str(excep): print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1002 but it has not.\n") def test_null_values(self): """ Test ID: DAOS-1376 Test Description: Pass a dkey and an akey that is null. :avocado: tags=all,full_regression,small :avocado: tags=object,objupdatebadparam :avocado: tags=objupdatenull """ # data used in the test thedata = b"a string that I want to stuff into an object" thedatasize = len(thedata) + 1 try: # try using a null dkey dkey = None akey = b"this is the akey" self.container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.plog.error("Didn't get expected return code.") self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): self.plog.error("Didn't get expected return code.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # try using a null akey/io descriptor dkey = b"this is the dkey" akey = None self.container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.fail("Test was expected to return a -1003 but it has not.\n") except DaosApiError as excep: if '-1003' not in str(excep): self.plog.error("Didn't get expected return code.") print(excep) print(traceback.format_exc()) self.fail("Test was expected to get -1003 but it has not.\n") try: # lastly try passing no data thedata = None thedatasize = 0 dkey = b"this is the dkey" akey = b"this is the akey" self.container.write_an_obj(thedata, thedatasize, dkey, akey, None, None, 2) self.plog.info("Update with no data worked") except DaosApiError as excep: print(excep) print(traceback.format_exc()) self.plog.error("Update with no data failed") self.fail("Update with no data failed.\n") self.plog.info("Test Complete")
class DeleteContainerTest(TestWithServers): """ Tests DAOS container delete and close. :avocado: recursive """ def test_container_delete(self): """ Test basic container delete :avocado: tags=all,container,tiny,smoke,full_regression,contdelete """ expected_for_param = [] uuidlist = self.params.get("uuid", '/run/createtests/ContainerUUIDS/*/') cont_uuid = uuidlist[0] expected_for_param.append(uuidlist[1]) pohlist = self.params.get("poh", '/run/createtests/PoolHandles/*/') poh = pohlist[0] expected_for_param.append(pohlist[1]) openlist = self.params.get("opened", "/run/createtests/ConnectionOpened/*/") opened = openlist[0] expected_for_param.append(openlist[1]) forcelist = self.params.get("force", "/run/createtests/ForceDestroy/*/") force = forcelist[0] # force=0 in .yaml file specifies FAIL, however: # if not opened and force=0 expect pass if force == 0 and not opened: expected_for_param.append('PASS') else: expected_for_param.append(forcelist[1]) # opened=True in .yaml file specifies PASS, however # if it is also the case force=0, then FAIL is expected expected_result = 'PASS' for result in expected_for_param: if result == 'FAIL': expected_result = 'FAIL' break # initialize a python pool object then create the underlying # daos storage and connect to it self.prepare_pool() passed = False try: self.container = DaosContainer(self.context) # create should always work (testing destroy) if not cont_uuid == 'INVALID': cont_uuid = uuid.UUID(uuidlist[0]) save_cont_uuid = cont_uuid self.container.create(self.pool.pool.handle, cont_uuid) else: self.container.create(self.pool.pool.handle) save_cont_uuid = uuid.UUID(self.container.get_uuid_str()) # Opens the container if required if opened: self.container.open(self.pool.pool.handle) # wait a few seconds and then attempts to destroy container time.sleep(5) if poh == 'VALID': poh = self.pool.pool.handle # if container is INVALID, overwrite with non existing UUID if cont_uuid == 'INVALID': cont_uuid = uuid.uuid4() self.container.destroy(force=force, poh=poh, con_uuid=cont_uuid) passed = True except DaosApiError as excep: self.log.info(excep, traceback.format_exc()) self.container.destroy(force=1, poh=self.pool.pool.handle, con_uuid=save_cont_uuid) finally: # close container handle, release a reference on pool in client lib # Otherwise test will ERROR in tearDown (pool disconnect -DER_BUSY) if opened: self.container.close() self.container = None if expected_result == 'PASS' and not passed: self.fail("Test was expected to pass but it failed.\n") if expected_result == 'FAIL' and passed: self.fail("Test was expected to fail but it passed.\n")