def verify_enspace_log(self, der_nospace_err_count):
"""
Function to verify there are no other error except DER_NOSPACE and
DER_NO_HDL in client log.Verify DER_NOSPACE count is higher.
args:
expected_err_count(int): Expected DER_NOSPACE count from client log.
"""
#Get the DER_NOSPACE and other error count from log
self.der_nospace_count, self.other_errors_count = error_count(
"-1007", self.hostlist_clients, self.client_log)
#Get the DER_NO_HDL and other error count from log
der_nohdl_count, other_nohdl_err = error_count("-1002",
self.hostlist_clients,
self.client_log)
#Check there are no other errors in log file except DER_NO_HDL
if self.other_errors_count != der_nohdl_count:
self.fail('Found other errors, count {} in client log {}'.format(
int(self.other_errors_count - other_nohdl_err),
self.client_log))
#Check the DER_NOSPACE error count is higher if not test will FAIL
if self.der_nospace_count < der_nospace_err_count:
self.fail(
'Expected DER_NOSPACE should be > {} and Found {}'.format(
der_nospace_err_count, self.der_nospace_count))
def run_enospace_with_bg_job(self):
"""
Function to run test and validate DER_ENOSPACE and expected storage
size. Single IOR job will run in background while space is filling.
"""
#Get the initial DER_ENOSPACE count
self.der_nospace_count, self.other_errors_count = error_count(
"-1007", self.hostlist_clients, self.client_log)
# Start the IOR Background thread which will write small data set and
# read in loop, until storage space is full.
job = threading.Thread(target=self.ior_bg_thread)
job.daemon = True
job.start()
#Run IOR in Foreground
self.run_enospace_foreground()
# Verify the background job result has no FAIL for any IOR run
for _result in self.test_result:
if "FAIL" in _result:
self.fail("One of the Background IOR job failed")
def run_enospace_with_bg_job(self):
"""
Function to run test and validate DER_ENOSPACE and expected storage
size. Single IOR job will run in background while space is filling.
"""
#Get the initial DER_ENOSPACE count
self.der_nospace_count, self.other_errors_count = error_count(
"-1007", self.hostlist_clients, self.client_log)
# Start the IOR Background thread which will write small data set and
# read in loop, until storage space is full.
out_queue = queue.Queue()
job = threading.Thread(target=self.ior_bg_thread,
kwargs={"results": out_queue})
job.daemon = True
job.start()
#Run IOR in Foreground
self.run_enospace_foreground()
# Verify the background job queue and make sure no FAIL for any IOR run
while not self.out_queue.empty():
if self.out_queue.get() == "FAIL":
self.fail("One of the Background IOR job failed")
def test_enospace_no_aggregation(self):
"""Jira ID: DAOS-4756.
Test Description: IO gets DER_NOSPACE when SCM is full and it release
the size when container destroy with Aggregation
disabled.
Use Case: This tests will create the pool and disable aggregation. Fill
75% of SCM size which should work, next try fill 10% more
which should fail with DER_NOSPACE. Destroy the container
and validate the Pool SCM free size is close to full (> 95%).
Do this in loop ~10 times and verify the DER_NOSPACE and SCM
free size after container destroy.
:avocado: tags=all,hw,medium,nvme,ib2,full_regression
:avocado: tags=der_enospace,enospc_no_aggregation
"""
# pylint: disable=attribute-defined-outside-init
# pylint: disable=too-many-branches
print(self.pool.pool_percentage_used())
# Disable the aggregation
self.pool.set_property("reclaim", "disabled")
#Get the DER_NOSPACE and other error count from log
self.der_nospace_count, self.other_errors_count = error_count(
"-1007", self.hostlist_clients, self.client_log)
#Repeat the test in loop.
for _loop in range(10):
print("-------enospc_no_aggregation Loop--------- {}".format(_loop))
#Fill 75% of SCM pool
self.start_ior_load(storage='SCM', percent=40)
print(self.pool.pool_percentage_used())
try:
#Fill 10% more to SCM ,which should Fail because no SCM space
self.start_ior_load(storage='SCM', percent=40)
self.fail('This test suppose to fail because of DER_NOSPACE'
'but it got Passed')
except TestFail as _error:
self.log.info('Expected to fail because of DER_NOSPACE')
#Verify DER_NO_SAPCE error count is expected and no other Error
#in client log.
self.verify_enspace_log(self.der_nospace_count)
#Delete all the containers
self.delete_all_containers()
#Get the pool usage
pool_usage = self.pool.pool_percentage_used()
#Delay to release the SCM size.
time.sleep(60)
print(pool_usage)
#SCM pool size should be released (some still be used for system)
#Pool SCM free % should not be less than 50%
if pool_usage['scm'] > 55:
self.fail('SCM pool used percentage should be < 55, instead {}'.
format(pool_usage['scm']))
#Run last IO
self.start_ior_load(storage='SCM', percent=1)
