def verify_enspace_log(self, der_nospace_err_count): """ Function to verify there are no other error except DER_NOSPACE and DER_NO_HDL in client log.Verify DER_NOSPACE count is higher. args: expected_err_count(int): Expected DER_NOSPACE count from client log. """ #Get the DER_NOSPACE and other error count from log self.der_nospace_count, self.other_errors_count = error_count( "-1007", self.hostlist_clients, self.client_log) #Get the DER_NO_HDL and other error count from log der_nohdl_count, other_nohdl_err = error_count("-1002", self.hostlist_clients, self.client_log) #Check there are no other errors in log file except DER_NO_HDL if self.other_errors_count != der_nohdl_count: self.fail('Found other errors, count {} in client log {}'.format( int(self.other_errors_count - other_nohdl_err), self.client_log)) #Check the DER_NOSPACE error count is higher if not test will FAIL if self.der_nospace_count < der_nospace_err_count: self.fail( 'Expected DER_NOSPACE should be > {} and Found {}'.format( der_nospace_err_count, self.der_nospace_count))
def run_enospace_with_bg_job(self): """ Function to run test and validate DER_ENOSPACE and expected storage size. Single IOR job will run in background while space is filling. """ #Get the initial DER_ENOSPACE count self.der_nospace_count, self.other_errors_count = error_count( "-1007", self.hostlist_clients, self.client_log) # Start the IOR Background thread which will write small data set and # read in loop, until storage space is full. job = threading.Thread(target=self.ior_bg_thread) job.daemon = True job.start() #Run IOR in Foreground self.run_enospace_foreground() # Verify the background job result has no FAIL for any IOR run for _result in self.test_result: if "FAIL" in _result: self.fail("One of the Background IOR job failed")
def run_enospace_with_bg_job(self): """ Function to run test and validate DER_ENOSPACE and expected storage size. Single IOR job will run in background while space is filling. """ #Get the initial DER_ENOSPACE count self.der_nospace_count, self.other_errors_count = error_count( "-1007", self.hostlist_clients, self.client_log) # Start the IOR Background thread which will write small data set and # read in loop, until storage space is full. out_queue = queue.Queue() job = threading.Thread(target=self.ior_bg_thread, kwargs={"results": out_queue}) job.daemon = True job.start() #Run IOR in Foreground self.run_enospace_foreground() # Verify the background job queue and make sure no FAIL for any IOR run while not self.out_queue.empty(): if self.out_queue.get() == "FAIL": self.fail("One of the Background IOR job failed")
def test_enospace_no_aggregation(self): """Jira ID: DAOS-4756. Test Description: IO gets DER_NOSPACE when SCM is full and it release the size when container destroy with Aggregation disabled. Use Case: This tests will create the pool and disable aggregation. Fill 75% of SCM size which should work, next try fill 10% more which should fail with DER_NOSPACE. Destroy the container and validate the Pool SCM free size is close to full (> 95%). Do this in loop ~10 times and verify the DER_NOSPACE and SCM free size after container destroy. :avocado: tags=all,hw,medium,nvme,ib2,full_regression :avocado: tags=der_enospace,enospc_no_aggregation """ # pylint: disable=attribute-defined-outside-init # pylint: disable=too-many-branches print(self.pool.pool_percentage_used()) # Disable the aggregation self.pool.set_property("reclaim", "disabled") #Get the DER_NOSPACE and other error count from log self.der_nospace_count, self.other_errors_count = error_count( "-1007", self.hostlist_clients, self.client_log) #Repeat the test in loop. for _loop in range(10): print("-------enospc_no_aggregation Loop--------- {}".format(_loop)) #Fill 75% of SCM pool self.start_ior_load(storage='SCM', percent=40) print(self.pool.pool_percentage_used()) try: #Fill 10% more to SCM ,which should Fail because no SCM space self.start_ior_load(storage='SCM', percent=40) self.fail('This test suppose to fail because of DER_NOSPACE' 'but it got Passed') except TestFail as _error: self.log.info('Expected to fail because of DER_NOSPACE') #Verify DER_NO_SAPCE error count is expected and no other Error #in client log. self.verify_enspace_log(self.der_nospace_count) #Delete all the containers self.delete_all_containers() #Get the pool usage pool_usage = self.pool.pool_percentage_used() #Delay to release the SCM size. time.sleep(60) print(pool_usage) #SCM pool size should be released (some still be used for system) #Pool SCM free % should not be less than 50% if pool_usage['scm'] > 55: self.fail('SCM pool used percentage should be < 55, instead {}'. format(pool_usage['scm'])) #Run last IO self.start_ior_load(storage='SCM', percent=1)