def write_single_object(self):
"""Write some data to the existing pool.
"""
self.pool.connect(2)
csum = self.params.get("enable_checksum", '/run/container/*')
container = DaosContainer(self.context)
input_param = container.cont_input_values
input_param.enable_chksum = csum
container.create(poh=self.pool.pool.handle,
con_prop=input_param)
container.open()
obj = DaosObj(self.context, container)
obj.create(objcls=1)
obj.open()
ioreq = IORequest(self.context,
container,
obj, objtype=4)
self.log.info("Writing the Single Dataset")
record_index = 0
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0])
* self.record_length[record_index])
d_key_value = "dkey {0}".format(dkey)
c_dkey = ctypes.create_string_buffer(d_key_value)
a_key_value = "akey {0}".format(akey)
c_akey = ctypes.create_string_buffer(a_key_value)
c_value = ctypes.create_string_buffer(indata)
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
ioreq.single_insert(c_dkey, c_akey, c_value, c_size)
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
def test_bad_handle(self):
"""
Test ID: DAOS-1376
Test Description: Pass a bogus object handle, should return bad handle.
:avocado: tags=all,object,full_regression,small,objbadhand
"""
self.prepare_pool()
try:
# create a container
container = DaosContainer(self.context)
container.create(self.pool.pool.handle)
self.plog.info("Container %s created.", container.get_uuid_str())
# now open it
container.open()
# create an object and write some data into it
thedata = "a string that I want to stuff into an object"
thedatasize = len(thedata) + 1
dkey = "this is the dkey"
akey = "this is the akey"
obj = container.write_an_obj(thedata, thedatasize, dkey, akey,
None, None, 2)
saved_oh = obj.obj_handle
obj.obj_handle = 99999
obj = container.write_an_obj(thedata, thedatasize, dkey, akey, obj,
None, 2)
container.oh = saved_oh
container.close()
container.destroy()
self.fail("Test was expected to return a -1002 but it has not.\n")
except DaosApiError as excep:
container.oh = saved_oh
container.close()
container.destroy()
self.plog.info("Test Complete")
if '-1002' not in str(excep):
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1002 but it has not.\n")
def get_container(context, pool, log=None):
"""Retrun a DAOS a container that has been created an opened.
Args:
context (DaosContext): the context to use to create the container
pool (DaosPool): pool in which to create the container
log (DaosLog|None): object for logging messages
Returns:
DaosContainer: an object representing a DAOS container
"""
if log:
log.info("Creating a container")
container = DaosContainer(context)
container.create(pool.handle)
if log:
log.info("Opening a container")
container.open()
return container
def test_array_obj(self):
"""
Test ID: DAOS-961
Test Description: Writes an array to an object and then reads it
back and verifies it.
:avocado: tags=all,smoke,pr,object,tiny,basicobject
"""
try:
# parameters used in pool create
createmode = self.params.get("mode",
'/run/pool_params/createmode/')
createsetid = self.params.get("setname",
'/run/pool_params/createset/')
createsize = self.params.get("size",
'/run/pool_params/createsize/')
createuid = os.geteuid()
creategid = os.getegid()
# initialize a python pool object then create the underlying
# daos storage
pool = DaosPool(self.context)
pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
self.plog.info("Pool %s created.", pool.get_uuid_str())
# need a connection to create container
pool.connect(1 << 1)
# create a container
container = DaosContainer(self.context)
container.create(pool.handle)
self.plog.info("Container %s created.", container.get_uuid_str())
# now open it
container.open()
# do a query and compare the UUID returned from create with
# that returned by query
container.query()
if container.get_uuid_str() != c_uuid_to_str(
container.info.ci_uuid):
self.fail("Container UUID did not match the one in info\n")
# create an object and write some data into it
thedata = []
thedata.append("data string one")
thedata.append("data string two")
thedata.append("data string tre")
dkey = "this is the dkey"
akey = "this is the akey"
self.plog.info("writing array to dkey >%s< akey >%s<.", dkey, akey)
oid, epoch = container.write_an_array_value(thedata,
dkey,
akey,
obj_cls=3)
# read the data back and make sure its correct
length = len(thedata[0])
thedata2 = container.read_an_array(len(thedata), length + 1, dkey,
akey, oid, epoch)
if thedata[0][0:length - 1] != thedata2[0][0:length - 1]:
self.plog.error("Data mismatch")
self.plog.error("Wrote: >%s<", thedata[0])
self.plog.error("Read: >%s<", thedata2[0])
self.fail("Write data, read it back, didn't match\n")
if thedata[2][0:length - 1] != thedata2[2][0:length - 1]:
self.plog.error("Data mismatch")
self.plog.error("Wrote: >%s<", thedata[2])
self.plog.error("Read: >%s<", thedata2[2])
self.fail("Write data, read it back, didn't match\n")
container.close()
# wait a few seconds and then destroy
time.sleep(5)
container.destroy()
# cleanup the pool
pool.disconnect()
pool.destroy(1)
self.plog.info("Test Complete")
except DaosApiError as excep:
self.plog.error("Test Failed, exception was thrown.")
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to pass but it failed.\n")
def test_null_values(self):
"""
Test ID: DAOS-1376
Test Description: Pass a dkey and an akey that is null.
:avocado: tags=all,object,full_regression,small,objupdatenull
"""
self.prepare_pool()
try:
# create a container
container = DaosContainer(self.context)
container.create(self.pool.pool.handle)
self.plog.info("Container %s created.", container.get_uuid_str())
# now open it
container.open()
# data used in the test
thedata = "a string that I want to stuff into an object"
thedatasize = len(thedata) + 1
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
self.fail("Test failed during setup .\n")
try:
# try using a null dkey
dkey = None
akey = "this is the akey"
container.write_an_obj(thedata, thedatasize, dkey, akey, None,
None, 2)
container.close()
container.destroy()
self.plog.error("Didn't get expected return code.")
self.fail("Test was expected to return a -1003 but it has not.\n")
except DaosApiError as excep:
if '-1003' not in str(excep):
container.close()
container.destroy()
self.plog.error("Didn't get expected return code.")
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1003 but it has not.\n")
try:
# try using a null akey/io descriptor
dkey = "this is the dkey"
akey = None
container.write_an_obj(thedata, thedatasize, dkey, akey, None,
None, 2)
self.fail("Test was expected to return a -1003 but it has not.\n")
except DaosApiError as excep:
if '-1003' not in str(excep):
self.plog.error("Didn't get expected return code.")
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1003 but it has not.\n")
try:
# lastly try passing no data
thedata = None
thedatasize = 0
dkey = "this is the dkey"
akey = "this is the akey"
container.write_an_obj(thedata, thedatasize, dkey, akey, None,
None, 2)
self.plog.info("Update with no data worked")
except DaosApiError as excep:
container.close()
container.destroy()
print(excep)
print(traceback.format_exc())
self.plog.error("Update with no data failed")
self.fail("Update with no data failed.\n")
container.close()
container.destroy()
self.plog.info("Test Complete")
class ContainerAttributeTest(TestWithServers):
"""
Tests DAOS container attribute get/set/list.
:avocado: recursive
"""
def setUp(self):
super(ContainerAttributeTest, self).setUp()
self.large_data_set = {}
self.prepare_pool()
poh = self.pool.pool.handle
self.container = DaosContainer(self.context)
self.container.create(poh)
self.container.open()
def create_data_set(self):
"""
To create the large attribute dictionary
"""
allchar = string.ascii_letters + string.digits
for i in range(1024):
self.large_data_set[str(i)] = (
"".join(random.choice(allchar)
for x in range(random.randint(1, 100))))
def test_container_large_attributes(self):
"""
Test ID: DAOS-1359
Test description: Test large randomly created container attribute.
:avocado: tags=container,container_attr,attribute,large_conattribute
"""
self.create_data_set()
attr_dict = self.large_data_set
try:
self.container.set_attr(data=attr_dict)
size, buf = self.container.list_attr()
verify_list_attr(attr_dict, size, buf)
results = {}
results = self.container.get_attr(attr_dict.keys())
verify_get_attr(attr_dict, results)
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to pass but it failed.\n")
def test_container_attribute(self):
"""
Test basic container attribute tests.
:avocado: tags=all,tiny,full_regression,container,sync_conattribute
"""
expected_for_param = []
name = self.params.get("name", '/run/attrtests/name_handles/*/')
expected_for_param.append(name[1])
value = self.params.get("value", '/run/attrtests/value_handles/*/')
expected_for_param.append(value[1])
attr_dict = {name[0]: value[0]}
expected_result = 'PASS'
for result in expected_for_param:
if result == 'FAIL':
expected_result = 'FAIL'
break
try:
self.container.set_attr(data=attr_dict)
size, buf = self.container.list_attr()
verify_list_attr(attr_dict, size, buf)
# Request something that doesn't exist
if name[0] is not None and "Negative" in name[0]:
name[0] = "rubbish"
results = {}
results = self.container.get_attr([name[0]])
verify_get_attr(attr_dict, results)
if expected_result in ['FAIL']:
self.fail("Test was expected to fail but it passed.\n")
except (DaosApiError, DaosTestError) as excep:
print(excep)
print(traceback.format_exc())
if expected_result == 'PASS':
self.fail("Test was expected to pass but it failed.\n")
def test_container_attribute_asyn(self):
"""
Test basic container attribute tests.
:avocado: tags=all,small,full_regression,container,async_conattribute
"""
global GLOB_SIGNAL
global GLOB_RC
expected_for_param = []
name = self.params.get("name", '/run/attrtests/name_handles/*/')
expected_for_param.append(name[1])
value = self.params.get("value", '/run/attrtests/value_handles/*/')
expected_for_param.append(value[1])
attr_dict = {name[0]: value[0]}
expected_result = 'PASS'
for result in expected_for_param:
if result == 'FAIL':
expected_result = 'FAIL'
break
try:
GLOB_SIGNAL = threading.Event()
self.container.set_attr(data=attr_dict, cb_func=cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC != 0 and expected_result in ['PASS']:
self.fail("RC not as expected after set_attr First {0}"
.format(GLOB_RC))
GLOB_SIGNAL = threading.Event()
size, buf = self.container.list_attr(cb_func=cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC != 0 and expected_result in ['PASS']:
self.fail("RC not as expected after list_attr First {0}"
.format(GLOB_RC))
if expected_result in ['PASS']:
verify_list_attr(attr_dict, size, buf, mode="async")
# Request something that doesn't exist
if name[0] is not None and "Negative" in name[0]:
name[0] = "rubbish"
GLOB_SIGNAL = threading.Event()
self.container.get_attr([name[0]], cb_func=cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC != 0 and expected_result in ['PASS']:
self.fail("RC not as expected after get_attr {0}"
.format(GLOB_RC))
# not verifying the get_attr since its not available asynchronously
if value[0] is not None:
if GLOB_RC == 0 and expected_result in ['FAIL']:
self.fail("Test was expected to fail but it passed.\n")
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
if expected_result == 'PASS':
self.fail("Test was expected to pass but it failed.\n")
def test_tx_basics(self):
"""
Perform I/O to an object in a container in 2 different transactions,
verifying basic I/O and transactions in particular.
NOTE: this was an epoch test and all I did was get it working with tx
Not a good test at this point, need to redesign when tx is fully
working.
:avocado: tags=all,container,tx,small,smoke,pr,basictx
"""
self.pool = None
try:
# parameters used in pool create
createmode = self.params.get("mode", '/run/poolparams/createmode/')
createuid = os.geteuid()
creategid = os.getegid()
createsetid = self.params.get("setname",
'/run/poolparams/createset/')
createsize = self.params.get("size", '/run/poolparams/createsize/')
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.pool.create(createmode, createuid, creategid,
createsize, createsetid, None)
# need a connection to create container
self.pool.connect(1 << 1)
# create a container
container = DaosContainer(self.context)
container.create(self.pool.handle)
# now open it
container.open()
# do a query and compare the UUID returned from create with
# that returned by query
container.query()
if container.get_uuid_str() != c_uuid_to_str(
container.info.ci_uuid):
self.fail("Container UUID did not match the one in info\n")
# create an object and write some data into it
thedata = "a string that I want to stuff into an object"
thedatasize = 45
dkey = "this is the dkey"
akey = "this is the akey"
oid, txn = container.write_an_obj(thedata, thedatasize,
dkey, akey, None, None, 2)
# read the data back and make sure its correct
thedata2 = container.read_an_obj(thedatasize, dkey, akey,
oid, txn)
if thedata != thedata2.value:
print("thedata>" + thedata)
print("thedata2>" + thedata2.value)
self.fail("Write data 1, read it back, didn't match\n")
# repeat above, but know that the write_an_obj call is advancing
# the epoch so the original copy remains and the new copy is in
# a new epoch.
thedata3 = "a different string"
thedatasize2 = 19
# note using the same keys so writing to the same spot
dkey = "this is the dkey"
akey = "this is the akey"
oid, tx2 = container.write_an_obj(thedata3, thedatasize2,
dkey, akey, oid, None, 2)
# read the data back and make sure its correct
thedata4 = container.read_an_obj(thedatasize2, dkey, akey,
oid, tx2)
if thedata3 != thedata4.value:
self.fail("Write data 2, read it back, didn't match\n")
# transactions generally don't work this way but need to explore
# an alternative to below code once model is complete, maybe
# read from a snapshot or read from TX_NONE etc.
# the original data should still be there too
#thedata5 = container.read_an_obj(thedatasize, dkey, akey,
# oid, transaction)
#if thedata != thedata5.value:
# self.fail("Write data 3, read it back, didn't match\n")
container.close()
# wait a few seconds and then destroy
time.sleep(5)
container.destroy()
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to pass but it failed.\n")
class CreateManyDkeys(TestWithServers):
"""
Test Class Description:
Tests that create large numbers of keys in objects/containers and then
destroy the containers and verify the space has been reclaimed.
:avocado: recursive
"""
def write_a_bunch_of_values(self, how_many):
"""
Write data to an object, each with a dkey and akey. The how_many
parameter determines how many key:value pairs are written.
"""
self.container = DaosContainer(self.context)
self.container.create(self.pool.pool.handle)
self.container.open()
ioreq = IORequest(self.context, self.container, None)
print("Started Writing the Dataset-----------\n")
inc = 50000
last_key = inc
for key in range(how_many):
c_dkey = create_string_buffer("dkey {0}".format(key))
c_akey = create_string_buffer("akey {0}".format(key))
c_value = create_string_buffer(
"some data that gets stored with the key {0}".format(key))
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
ioreq.single_insert(c_dkey,
c_akey,
c_value,
c_size)
if key > last_key:
print("written: {}".format(key))
sys.stdout.flush()
last_key = key + inc
print("Started Verification of the Dataset-----------\n")
last_key = inc
for key in range(how_many):
c_dkey = create_string_buffer("dkey {0}".format(key))
c_akey = create_string_buffer("akey {0}".format(key))
the_data = "some data that gets stored with the key {0}".format(key)
val = ioreq.single_fetch(c_dkey,
c_akey,
len(the_data)+1)
exp_value = val.value.decode("utf-8")
if the_data != exp_value:
self.fail("ERROR: Data mismatch for dkey = {0}, akey={1}, "
"Expected Value={2} and Received Value={3}\n"
.format("dkey {0}".format(key),
"akey {0}".format(key),
the_data,
exp_value))
if key > last_key:
print("veried: {}".format(key))
sys.stdout.flush()
last_key = key + inc
print("starting destroy")
self.container.close()
self.container.destroy()
print("destroy complete")
@avocado.fail_on(DaosApiError)
def test_many_dkeys(self):
"""
Test ID: DAOS-1701
Test Description: Test many of dkeys in same object.
Use Cases: 1. large key counts
2. space reclamation after destroy
:avocado: tags=all,full_regression
:avocado: tags=small
:avocado: tags=object
:avocado: tags=many_dkeys
"""
self.prepare_pool()
no_of_dkeys = self.params.get("number_of_dkeys", '/run/dkeys/')
# write a lot of individual data items, verify them, then destroy
self.write_a_bunch_of_values(no_of_dkeys)
# do it again, which should verify the first container
# was truly destroyed because a second round won't fit otherwise
self.write_a_bunch_of_values(no_of_dkeys)
class CsumContainerValidation(TestWithServers):
"""
Test Class Description: This test is enables
checksum container properties and performs
single object inserts and verifies
contents. This is a basic sanity
test for enabling checksum testing.
:avocado: recursive
"""
# pylint: disable=too-many-instance-attributes
def setUp(self):
super().setUp()
self.agent_sessions = None
self.pool = None
self.container = None
self.obj = None
self.ioreq = None
self.no_of_dkeys = None
self.no_of_akeys = None
self.array_size = None
self.record_length = None
self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0]
self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0]
self.record_length = self.params.get("length", '/run/record/*')
self.add_pool(connect=False)
self.pool.connect(2)
self.csum = self.params.get("enable_checksum", '/run/container/*')
self.container = DaosContainer(self.context)
input_param = self.container.cont_input_values
input_param.enable_chksum = self.csum
self.container.create(poh=self.pool.pool.handle,
con_prop=input_param)
self.container.open()
self.obj = DaosObj(self.context, self.container)
self.obj.create(objcls=1)
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj, objtype=4)
def test_single_object_with_checksum(self):
"""
Test ID: DAOS-3927
Test Description: Write Avocado Test to verify single data after
pool/container disconnect/reconnect.
:avocado: tags=all,daily_regression
:avocado: tags=checksum
:avocado: tags=basic_checksum_object
"""
self.d_log.info("Writing the Single Dataset")
record_index = 0
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0])
* self.record_length[record_index])
c_dkey = create_string_buffer("dkey {0}".format(dkey))
c_akey = create_string_buffer("akey {0}".format(akey))
c_value = create_string_buffer(indata)
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size)
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
self.d_log.info("Single Dataset Verification -- Started")
record_index = 0
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
c_dkey = create_string_buffer("dkey {0}".format(dkey))
c_akey = create_string_buffer("akey {0}".format(akey))
val = self.ioreq.single_fetch(c_dkey,
c_akey,
len(indata)+1)
if indata != val.value.decode('utf-8'):
message = (
"ERROR:Data mismatch for dkey={}, akey={}: indata={}, "
"val={}".format(
dkey, akey, indata, val.value.decode('utf-8')))
self.d_log.error(message)
self.fail(message)
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
class SameKeyDifferentValue(TestWithServers):
"""
Test Description: Test to verify different type of values
passed to same akey and dkey.
:avocado: recursive
"""
def setUp(self):
super(SameKeyDifferentValue, self).setUp()
self.prepare_pool()
try:
# create a container
self.container = DaosContainer(self.context)
self.container.create(self.pool.pool.handle)
# now open it
self.container.open()
except DaosApiError as excpn:
print(excpn)
print(traceback.format_exc())
self.fail("Test failed during setup.\n")
def test_single_to_array_value(self):
"""
Jira ID: DAOS-2218
Test Description: Test to verify different type of
values passed (i.e. single to array value) to the same akey and dkey.
Case1: Insert akey,dkey with single value
Insert same akey,dkey with array value
Result: should return -1001 ERR.
Case2: Insert akey,dkey with single value
Punch the keys
Insert same akey,dkey under same object with array value
Result: should either pass or return -1001 ERR
Case3: Insert akey,dkey with single value
Punch the keys
Trigger aggregation
Insert same akey,dkey under same object with array value
Result: should either pass or return -1001 ERR
:avocado: tags=object,samekeydifferentvalue,singletoarray,vm,small
"""
# define akey,dkey, single value data and array value data
single_value_data = "a string that I want to stuff into an object"
array_value_data = []
array_value_data.append("data string one")
array_value_data.append("data string two")
array_value_data.append("data string tre")
dkey = "this is the dkey"
akey = "this is the akey"
aggregation = False
for i in range(3):
try:
# create an object and write single value data into it
obj = self.container.write_an_obj(single_value_data,
len(single_value_data) + 1,
dkey,
akey,
obj_cls=1)
# read the data back and make sure its correct
read_back_data = self.container.read_an_obj(
len(single_value_data) + 1, dkey, akey, obj)
if single_value_data != read_back_data.value:
print("data I wrote:" + single_value_data)
print("data I read back" + read_back_data.value)
self.fail("Write data, read it back, didn't match\n")
# test case 1
if i == 0:
try:
# write array value data to same keys, expected to fail
self.container.write_an_array_value(array_value_data,
dkey,
akey,
obj,
obj_cls=1)
# above line is expected to return an error,
# if not fail the test
self.fail(
"Array value write to existing single value" +
" key should have failed\n")
# should fail with -1001 ERR
except DaosApiError as excp:
if "-1001" not in str(excp):
print(excp)
self.fail("Should have failed with -1001 error" +
" message, but it did not\n")
# test case 2 and 3
elif i == 1 or 2:
try:
# punch the keys
obj.punch_akeys(0, dkey, [akey])
obj.punch_dkeys(0, [dkey])
if aggregation is True:
# trigger aggregation
self.container.aggregate(self.container.coh, 0)
# write to the same set of keys under same object
# with array value type
self.container.write_an_array_value(array_value_data,
dkey,
akey,
obj,
obj_cls=1)
# above write of array value should either succeed
# or fail with -1001 ERR
except DaosApiError as excp:
if "-1001" not in str(excp):
print(excp)
self.fail("Should have failed with -1001 error" +
" message or the write should have" +
" been successful, but it did not\n")
# change the value of aggregation to test Test Case 3
aggregation = True
# punch the entire object after each iteration
obj.close()
# catch the exception if test fails to write to an object
# or fails to punch the written object
except DaosApiError as excp:
self.fail("Failed to write to akey/dkey or punch the object")
def test_array_to_single_value(self):
"""
Jira ID: DAOS-2218
Test Description: Test to verify different type of
values passed (i.e array to single value) to the same akey and dkey.
Case1: Insert akey,dkey with array value
Insert same akey,dkey with single value
Result: should return -1001 ERR.
Case2: Insert akey,dkey with array value
Punch the keys
Insert same akey,dkey under same object with single value
Result: should either pass or return -1001 ERR
Case3: Insert akey,dkey with array value
Punch the keys
Trigger aggregation
Insert same akey,dkey under same object with single value
Result: should either pass or return -1001 ERR
:avocado: tags=object,samekeydifferentvalue,arraytosingle,vm,small
"""
# define akey,dkey, single value data and array value data
single_value_data = "a string that I want to stuff into an object"
array_value_data = []
array_value_data.append("data string one")
array_value_data.append("data string two")
array_value_data.append("data string tre")
dkey = "this is the dkey"
akey = "this is the akey"
aggregation = False
for i in range(3):
try:
# create an object and write array value data into it
obj = self.container.write_an_array_value(array_value_data,
dkey,
akey,
obj_cls=1)
# read the data back and make sure its correct
length = len(array_value_data[0])
read_back_data = self.container.read_an_array(
len(array_value_data), length + 1, dkey, akey, obj)
for j in range(3):
if (array_value_data[j][0:length - 1] !=
read_back_data[j][0:length - 1]):
print("Written Data: {}".format(array_value_data[j]))
print("Read Data: {}".format(read_back_data[j]))
self.fail("Data mismatch\n")
# test case 1
if i == 0:
try:
# write single value data to same keys, expected to fail
self.container.write_an_obj(single_value_data,
len(single_value_data) + 1,
dkey,
akey,
obj,
obj_cls=1)
# above line is expected to return an error,
# if not fail the test
self.fail(
"Single value write to existing array value" +
" key should have failed\n")
# should fail with -1001 ERR
except DaosApiError as excp:
if "-1001" not in str(excp):
print(excp)
self.fail("Should have failed with -1001 error" +
" message, but it did not\n")
# test case 2 and 3
elif i == 1 or 2:
try:
# punch the keys
obj.punch_akeys(0, dkey, [akey])
obj.punch_dkeys(0, [dkey])
if aggregation is True:
# trigger aggregation
self.container.aggregate(self.container.coh, 0)
# write to the same set of keys under same object
# with single value type
self.container.write_an_obj(single_value_data,
len(single_value_data) + 1,
dkey,
akey,
obj,
obj_cls=1)
# above write of array value should either succeed
# or fail with -1001 ERR
except DaosApiError as excp:
if "-1001" not in str(excp):
print(excp)
self.fail("Should have failed with -1001 error" +
" message or the write should have" +
" been successful, but it did not\n")
# change the value of aggregation to test Test Case 3
aggregation = True
# punch the entire object after each iteration
obj.close()
# catch the exception if test fails to write to an object
# or fails to punch the written object
except DaosApiError as excp:
self.fail("Failed to write to akey/dkey or punch the object")
class ObjOpenBadParam(TestWithServers):
"""
Test Class Description:
Pass an assortment of bad parameters to the daos_obj_open function.
:avocado: recursive
"""
def setUp(self):
super(ObjOpenBadParam, self).setUp()
self.prepare_pool()
try:
# create a container
self.container = DaosContainer(self.context)
self.container.create(self.pool.pool.handle)
# now open it
self.container.open()
# create an object and write some data into it
thedata = "a string that I want to stuff into an object"
self.datasize = len(thedata) + 1
self.dkey = "this is the dkey"
self.akey = "this is the akey"
self.obj = self.container.write_an_obj(thedata,
self.datasize,
self.dkey,
self.akey,
obj_cls=1)
thedata2 = self.container.read_an_obj(self.datasize, self.dkey,
self.akey, self.obj)
if thedata not in thedata2.value:
print(thedata)
print(thedata2.value)
err_str = "Error reading back data, test failed during the " \
"initial setup."
self.d_log.error(err_str)
self.fail(err_str)
# setup leaves object in open state, so closing to start clean
self.obj.close()
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
self.fail("Test failed during the initial setup.")
def test_bad_obj_handle(self):
"""
Test ID: DAOS-1320
Test Description: Attempt to open a garbage object handle.
:avocado: tags=all,object,full_regression,tiny,objopenbadhandle
"""
saved_handle = self.obj.obj_handle
self.obj.obj_handle = 8675309
try:
dummy_obj = self.obj.open()
except DaosApiError as excep:
if '-1002' not in str(excep):
self.d_log.error("test expected a -1002 but did not get it")
self.d_log.error(traceback.format_exc())
self.fail("test expected a -1002 but did not get it")
finally:
self.obj.obj_handle = saved_handle
def test_invalid_container_handle(self):
"""
Test ID: DAOS-1320
Test Description: Attempt to open an object with a garbage container
handle.
:avocado: tags=all,object,full_regression,tiny,objopenbadcont
"""
saved_coh = self.container.coh
self.container.coh = 8675309
try:
dummy_obj = self.obj.open()
except DaosApiError as excep:
if '-1002' not in str(excep):
self.d_log.error("test expected a -1002 but did not get it")
self.d_log.error(traceback.format_exc())
self.fail("test expected a -1002 but did not get it")
finally:
self.container.coh = saved_coh
def test_closed_container_handle(self):
"""
Test ID: DAOS-1320
Test Description: Attempt to open an object in a container with
a closed handle.
:avocado: tags=all,object,full_regression,tiny,objopenclosedcont
"""
self.container.close()
try:
dummy_obj = self.obj.open()
except DaosApiError as excep:
if '-1002' not in str(excep):
self.d_log.error("test expected a -1002 but did not get it")
self.d_log.error(traceback.format_exc())
self.fail("test expected a -1002 but did not get it")
finally:
self.container.open()
def test_pool_handle_as_obj_handle(self):
"""
Test ID: DAOS-1320
Test Description: Adding this test by request, this test attempts
to open an object that's had its handle set to
be the same as a valid pool handle.
:avocado: tags=all,object,full_regression,tiny,objopenbadpool
"""
saved_oh = self.obj.obj_handle
self.obj.obj_handle = self.pool.pool.handle
try:
dummy_obj = self.obj.open()
except DaosApiError as excep:
if '-1002' not in str(excep):
self.d_log.error("test expected a -1002 but did not get it")
self.d_log.error(traceback.format_exc())
self.fail("test expected a -1002 but did not get it")
finally:
self.obj.obj_handle = saved_oh
def test_null_ranklist(self):
"""
Test ID: DAOS-1320
Test Description: Attempt to open an object in a container with
an empty ranklist.
:avocado: tags=all,object,full_regression,tiny,objopennullrl
"""
# null rl
saved_rl = self.obj.tgt_rank_list
self.obj.tgt_rank_list = None
try:
dummy_obj = self.obj.open()
except DaosApiError as excep:
if '-1003' not in str(excep):
self.d_log.error("test expected a -1003 but did not get it")
self.d_log.error(traceback.format_exc())
self.fail("test expected a -1003 but did not get it")
finally:
self.obj.tgt_rank_list = saved_rl
def test_null_oid(self):
"""
Test ID: DAOS-1320
Test Description: Attempt to open an object in a container with
null object id.
:avocado: tags=all,object,full_regression,tiny,objopennulloid
"""
# null oid
saved_oid = self.obj.c_oid
self.obj.c_oid = DaosObjId(0, 0)
try:
dummy_obj = self.obj.open()
except DaosApiError as excep:
if '-1003' not in str(excep):
self.d_log.error("Test expected a -1003 but did not get it")
self.d_log.error(traceback.format_exc())
self.fail("test expected a -1003 but did not get it")
finally:
self.obj.c_oid = saved_oid
def test_null_tgts(self):
"""
Test ID: DAOS-1320
Test Description: Attempt to open an object in a container with
null tgt.
:avocado: tags=all,object,full_regression,tiny,objopennulltgts
"""
# null tgts
saved_ctgts = self.obj.c_tgts
self.obj.c_tgts = 0
try:
dummy_obj = self.obj.open()
except DaosApiError as excep:
if '-1003' not in str(excep):
self.d_log.error("Test expected a -1003 but did not get it")
self.d_log.error(traceback.format_exc())
self.fail("test expected a -1003 but did not get it")
finally:
self.obj.c_tgts = saved_ctgts
def test_null_attrs(self):
"""
Test ID: DAOS-1320
Test Description: Attempt to open an object in a container with
null object attributes.
:avocado: tags=all,object,full_regression,tiny,objopennullattr
"""
# null attr
saved_attr = self.obj.attr
self.obj.attr = 0
try:
dummy_obj = self.obj.open()
except DaosApiError as excep:
if '-1003' not in str(excep):
self.d_log.error("test expected a -1003 but did not get it")
self.d_log.error(traceback.format_exc())
self.fail("test expected a -1003 but did not get it")
finally:
self.obj.attr = saved_attr
def test_null_values(self):
"""
Test ID: DAOS-1376
Test Description: Pass a dkey and an akey that is null.
:avocado: tags=all,object,full_regression,small,objupdatenull
"""
try:
# parameters used in pool create
createmode = self.params.get("mode", '/run/conttests/createmode/')
createsetid = self.params.get("setname",
'/run/conttests/createset/')
createsize = self.params.get("size", '/run/conttests/createsize/')
createuid = os.geteuid()
creategid = os.getegid()
# initialize a python pool object then create the underlying
# daos storage
pool = DaosPool(self.context)
pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
self.plog.info("Pool %s created.", pool.get_uuid_str())
# need a connection to create container
pool.connect(1 << 1)
# create a container
container = DaosContainer(self.context)
container.create(pool.handle)
self.plog.info("Container %s created.", container.get_uuid_str())
# now open it
container.open()
# data used in the test
thedata = "a string that I want to stuff into an object"
thedatasize = len(thedata) + 1
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
self.fail("Test failed during setup .\n")
try:
# try using a null dkey
dkey = None
akey = "this is the akey"
container.write_an_obj(thedata, thedatasize, dkey, akey, None,
None, 2)
container.close()
container.destroy()
pool.disconnect()
pool.destroy(1)
self.plog.error("Didn't get expected return code.")
self.fail("Test was expected to return a -1003 but it has not.\n")
except DaosApiError as excep:
if '-1003' not in str(excep):
container.close()
container.destroy()
pool.disconnect()
pool.destroy(1)
self.plog.error("Didn't get expected return code.")
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1003 but it has not.\n")
try:
# try using a null akey/io descriptor
dkey = "this is the dkey"
akey = None
container.write_an_obj(thedata, thedatasize, dkey, akey, None,
None, 2)
self.fail("Test was expected to return a -1003 but it has not.\n")
except DaosApiError as excep:
if '-1003' not in str(excep):
self.plog.error("Didn't get expected return code.")
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1003 but it has not.\n")
try:
# lastly try passing no data
thedata = None
thedatasize = 0
dkey = "this is the dkey"
akey = "this is the akey"
container.write_an_obj(thedata, thedatasize, dkey, akey, None,
None, 2)
self.plog.info("Update with no data worked")
except DaosApiError as excep:
container.close()
container.destroy()
pool.disconnect()
pool.destroy(1)
print(excep)
print(traceback.format_exc())
self.plog.error("Update with no data failed")
self.fail("Update with no data failed.\n")
container.close()
container.destroy()
pool.disconnect()
pool.destroy(1)
self.plog.info("Test Complete")
def test_bad_handle(self):
"""
Test ID: DAOS-1376
Test Description: Pass a bogus object handle, should return bad handle.
:avocado: tags=all,object,full_regression,small,objbadhand
"""
try:
# parameters used in pool create
createmode = self.params.get("mode", '/run/conttests/createmode/')
createsetid = self.params.get("setname",
'/run/conttests/createset/')
createsize = self.params.get("size", '/run/conttests/createsize/')
createuid = os.geteuid()
creategid = os.getegid()
# initialize a python pool object then create the underlying
# daos storage
pool = DaosPool(self.context)
pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
self.plog.info("Pool %s created.", pool.get_uuid_str())
# need a connection to create container
pool.connect(1 << 1)
# create a container
container = DaosContainer(self.context)
container.create(pool.handle)
self.plog.info("Container %s created.", container.get_uuid_str())
# now open it
container.open()
# create an object and write some data into it
thedata = "a string that I want to stuff into an object"
thedatasize = len(thedata) + 1
dkey = "this is the dkey"
akey = "this is the akey"
obj, dummy_tx = container.write_an_obj(thedata, thedatasize, dkey,
akey, None, None, 2)
saved_oh = obj.obj_handle
obj.obj_handle = 99999
obj, dummy_tx = container.write_an_obj(thedata, thedatasize, dkey,
akey, obj, None, 2)
container.oh = saved_oh
container.close()
container.destroy()
pool.disconnect()
pool.destroy(1)
self.fail("Test was expected to return a -1002 but it has not.\n")
except DaosApiError as excep:
container.oh = saved_oh
container.close()
container.destroy()
pool.disconnect()
pool.destroy(1)
self.plog.info("Test Complete")
if '-1002' not in str(excep):
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1002 but it has not.\n")
class Permission(TestWithServers):
"""Test pool permissions.
Test Class Description:
Tests DAOS pool permissions while connect, whether
modifying file with specific permissions work as expected.
:avocado: recursive
"""
def test_connectpermission(self):
"""Test ID: DAOS-???.
Test Description:
Test pool connections with specific permissions.
:avocado: tags=pool,permission,connectpermission
"""
# parameters used in pool create
createmode = self.params.get("mode", '/run/createtests/createmode/*/')
createuid = os.geteuid()
creategid = os.getegid()
createsetid = self.params.get("setname", '/run/createtests/createset/')
createsize = self.params.get("size", '/run/createtests/createsize/')
# parameters used for pool connect
permissions = self.params.get("perm", '/run/createtests/permissions/*')
if createmode == 73 or \
(createmode == 146 and permissions != 1) or \
(createmode == 292 and permissions != 3):
self.cancelForTicket("DAOS-3442")
if createmode == 73:
expected_result = 'FAIL'
if createmode == 511 and permissions == 0:
expected_result = 'PASS'
elif createmode in [146, 511] and permissions == 1:
expected_result = 'PASS'
elif createmode in [292, 511] and permissions == 2:
expected_result = 'PASS'
else:
expected_result = 'FAIL'
try:
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.multi_log("Pool initialisation successful", "debug")
self.pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
self.multi_log("Pool Creation successful", "debug")
self.pool.connect(1 << permissions)
self.multi_log("Pool Connect successful", "debug")
if expected_result in ['FAIL']:
self.fail("Test was expected to fail but it passed.\n")
except DaosApiError as excep:
self.log.error(str(excep))
if expected_result == 'PASS':
self.fail("Test was expected to pass but it failed.\n")
def test_filemodification(self):
"""Test ID: DAOS-???.
Test Description:
Test whether file modification happens as expected under different
permission levels.
:avocado: tags=pool,permission,filemodification
"""
# parameters used in pool create
createmode = self.params.get("mode", '/run/createtests/createmode/*/')
createuid = self.params.get("uid", '/run/createtests/createuid/')
creategid = self.params.get("gid", '/run/createtests/creategid/')
createsetid = self.params.get("setname", '/run/createtests/createset/')
createsize = self.params.get("size", '/run/createtests/createsize/')
if createmode == 73:
expected_result = 'FAIL'
elif createmode in [146, 511]:
permissions = 1
expected_result = 'PASS'
elif createmode == 292:
permissions = 2
expected_result = 'PASS'
try:
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.multi_log("Pool initialisation successful", "debug")
self.pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
self.multi_log("Pool Creation successful", "debug")
self.pool.connect(1 << permissions)
self.multi_log("Pool Connect successful", "debug")
self.container = DaosContainer(self.context)
self.multi_log("Contianer initialisation successful", "debug")
self.container.create(self.pool.handle)
self.multi_log("Container create successful", "debug")
# now open it
self.container.open()
self.multi_log("Container open successful", "debug")
thedata = "a string that I want to stuff into an object"
size = 45
dkey = "this is the dkey"
akey = "this is the akey"
self.container.write_an_obj(thedata, size, dkey, akey)
self.multi_log("Container write successful", "debug")
if expected_result in ['FAIL']:
self.fail("Test was expected to fail but it passed.\n")
except DaosApiError as excep:
self.log.error(str(excep))
if expected_result == 'PASS':
self.fail("Test was expected to pass but it failed.\n")
class ContainerAsync(TestWithServers):
"""
Tests DAOS pool connect permissions (non existing pool handle, bad uuid)
and close.
:avocado: recursive
"""
def __init__(self, *args, **kwargs):
super(ContainerAsync, self).__init__(*args, **kwargs)
self.container1 = None
self.container2 = None
self.pool = None
def test_createasync(self):
"""
Test container create for asynchronous mode.
:avocado: tags=all,small,full_regression,container,createasync
"""
global GLOB_SIGNAL
global GLOB_RC
# parameters used in pool create
createmode = self.params.get("mode", '/run/createtests/createmode/*/')
createsetid = self.params.get("setname", '/run/createtests/createset/')
createsize = self.params.get("size", '/run/createtests/createsize/')
createuid = os.geteuid()
creategid = os.getegid()
try:
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
poh = self.pool.handle
self.pool.connect(1 << 1)
# Container initialization and creation
self.container1 = DaosContainer(self.context)
self.container2 = DaosContainer(self.context)
GLOB_SIGNAL = threading.Event()
self.container1.create(poh, None, cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC != 0:
self.fail("RC not as expected in async test")
print("RC after successful container create: ", GLOB_RC)
# Try to recreate container after destroying pool,
# this should fail. Checking rc after failure.
self.pool.disconnect()
self.pool.destroy(1)
GLOB_SIGNAL = threading.Event()
GLOB_RC = -9900000
self.container2.create(poh, None, cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC == 0:
self.fail("RC not as expected in async test")
print("RC after unsuccessful container create: ", GLOB_RC)
# cleanup the pool and container
self.pool = None
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
def test_destroyasync(self):
"""
Test container destroy for asynchronous mode.
:avocado: tags=all,small,full_regression,container,contdestroyasync
"""
global GLOB_SIGNAL
global GLOB_RC
# parameters used in pool create
createmode = self.params.get("mode", '/run/createtests/createmode/*/')
createsetid = self.params.get("setname", '/run/createtests/createset/')
createsize = self.params.get("size", '/run/createtests/createsize/')
createuid = os.geteuid()
creategid = os.getegid()
try:
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
poh = self.pool.handle
self.pool.connect(1 << 1)
# Container initialization and creation
self.container1 = DaosContainer(self.context)
self.container2 = DaosContainer(self.context)
self.container1.create(poh)
GLOB_SIGNAL = threading.Event()
self.container1.destroy(1, poh, None, cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC != 0:
self.fail("RC not as expected in async test")
print("RC after successful container create: ", GLOB_RC)
# Try to destroy container again, this should fail, as non-existent.
# Checking rc after failure.
GLOB_SIGNAL = threading.Event()
GLOB_RC = -9900000
self.container2.destroy(1, poh, None, cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC != -1003:
self.fail("RC not as expected in async test")
print("RC after container destroy failed:", GLOB_RC)
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
def test_openasync(self):
"""
Test container open for asynchronous mode.
:avocado: tags=all,small,full_regression,container,openasync
"""
global GLOB_SIGNAL
global GLOB_RC
# parameters used in pool create
createmode = self.params.get("mode", '/run/createtests/createmode/*/')
createsetid = self.params.get("setname", '/run/createtests/createset/')
createsize = self.params.get("size", '/run/createtests/createsize/')
createuid = os.geteuid()
creategid = os.getegid()
try:
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
poh = self.pool.handle
self.pool.connect(1 << 1)
# Container initialization and creation
self.container1 = DaosContainer(self.context)
self.container2 = DaosContainer(self.context)
self.container1.create(poh)
str_cuuid = self.container1.get_uuid_str()
cuuid = uuid.UUID(str_cuuid)
GLOB_SIGNAL = threading.Event()
self.container1.open(poh, cuuid, 2, cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC != 0:
self.fail("RC not as expected in async test")
print("RC after successful container create: ", GLOB_RC)
# Try to open container2, this should fail, as non-existent.
# Checking rc after failure.
GLOB_SIGNAL = threading.Event()
GLOB_RC = -9900000
self.container2.open(None, None, None, cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC == 0:
self.fail("RC not as expected in async test")
print("RC after container destroy failed:", GLOB_RC)
# cleanup the container
self.container1.close()
self.container1.destroy()
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
def test_closeasync(self):
"""
Test container close for asynchronous mode.
:avocado: tags=all,small,full_regression,container,closeasync
"""
global GLOB_SIGNAL
global GLOB_RC
# parameters used in pool create
createmode = self.params.get("mode", '/run/createtests/createmode/*/')
createsetid = self.params.get("setname", '/run/createtests/createset/')
createsize = self.params.get("size", '/run/createtests/createsize/')
createuid = os.geteuid()
creategid = os.getegid()
try:
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
poh = self.pool.handle
self.pool.connect(1 << 1)
# Container initialization and creation
self.container1 = DaosContainer(self.context)
self.container2 = DaosContainer(self.context)
self.container1.create(poh)
str_cuuid = self.container1.get_uuid_str()
cuuid = uuid.UUID(str_cuuid)
self.container1.open(poh, cuuid, 2)
GLOB_SIGNAL = threading.Event()
self.container1.close(cb_func=cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC != 0:
self.fail("RC not as expected in async test: "
"{0}".format(GLOB_RC))
print("RC after successful container create: ", GLOB_RC)
# Try to open container2, this should fail, as non-existent.
# Checking rc after failure.
GLOB_SIGNAL = threading.Event()
GLOB_RC = -9900000
self.container2.close(cb_func=cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC == 0:
self.fail("RC not as expected in async test: "
"{0}".format(GLOB_RC))
print("RC after container destroy failed:", GLOB_RC)
# cleanup the container
self.container1.destroy()
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
def test_queryasync(self):
"""
Test container query for asynchronous mode.
:avocado: tags=all,small,full_regression,container,queryasync
"""
global GLOB_SIGNAL
global GLOB_RC
# parameters used in pool create
createmode = self.params.get("mode", '/run/createtests/createmode/*/')
createsetid = self.params.get("setname", '/run/createtests/createset/')
createsize = self.params.get("size", '/run/createtests/createsize/')
createuid = os.geteuid()
creategid = os.getegid()
try:
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
poh = self.pool.handle
self.pool.connect(1 << 1)
# Container initialization and creation
self.container1 = DaosContainer(self.context)
self.container2 = DaosContainer(self.context)
self.container1.create(poh)
dummy_str_cuuid = self.container1.get_uuid_str()
# Open container
self.container1.open(poh, None, 2, None)
GLOB_SIGNAL = threading.Event()
self.container1.query(cb_func=cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC != 0:
self.fail("RC not as expected in async test: "
"{0}".format(GLOB_RC))
print("RC after successful container create: ", GLOB_RC)
# Close opened container
self.container1.close()
# Try to open container2, this should fail, as non-existent.
# Checking rc after failure.
GLOB_SIGNAL = threading.Event()
GLOB_RC = -9900000
self.container2.query(cb_func=cb_func)
GLOB_SIGNAL.wait()
if GLOB_RC == 0:
self.fail("RC not as expected in async test: "
"{0}".format(GLOB_RC))
print("RC after container destroy failed:", GLOB_RC)
# cleanup the container
self.container1.destroy()
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
class OSAUtils(MdtestBase, IorTestBase):
# pylint: disable=too-many-ancestors
"""
Test Class Description: This test runs
daos_server offline drain test cases.
:avocado: recursive
"""
def setUp(self):
"""Set up for test case."""
super().setUp()
self.pool_cont_dict = {}
self.container = None
self.obj = None
self.ioreq = None
self.dmg_command = self.get_dmg_command()
self.no_of_dkeys = self.params.get("no_of_dkeys",
'/run/dkeys/*',
default=[0])[0]
self.no_of_akeys = self.params.get("no_of_akeys",
'/run/akeys/*',
default=[0])[0]
self.record_length = self.params.get("length",
'/run/record/*',
default=[0])[0]
self.ior_w_flags = self.params.get("write_flags",
'/run/ior/iorflags/*',
default="")
self.ior_r_flags = self.params.get("read_flags", '/run/ior/iorflags/*')
self.out_queue = queue.Queue()
self.dmg_command.exit_status_exception = False
self.test_during_aggregation = False
self.test_during_rebuild = False
self.test_with_checksum = True
@fail_on(CommandFailure)
def get_pool_leader(self):
"""Get the pool leader.
Returns:
int: pool leader value
"""
data = self.dmg_command.pool_query(self.pool.uuid)
return int(data["response"]["leader"])
@fail_on(CommandFailure)
def get_rebuild_status(self):
"""Get the rebuild status.
Returns:
str: reuild status
"""
data = self.dmg_command.pool_query(self.pool.uuid)
return data["response"]["rebuild"]["status"]
@fail_on(CommandFailure)
def is_rebuild_done(self,
time_interval,
wait_for_rebuild_to_complete=False):
"""Rebuild is completed/done.
Args:
time_interval: Wait interval between checks
wait_for_rebuild_to_complete: Rebuild completed
(Default: False)
"""
self.pool.wait_for_rebuild(wait_for_rebuild_to_complete,
interval=time_interval)
@fail_on(CommandFailure)
def assert_on_rebuild_failure(self):
"""If the rebuild is not successful,
raise assert.
"""
rebuild_status = self.get_rebuild_status()
self.log.info("Rebuild Status: %s", rebuild_status)
rebuild_failed_string = ["failed", "scanning", "aborted", "busy"]
self.assertTrue(rebuild_status not in rebuild_failed_string,
"Rebuild failed")
@fail_on(CommandFailure)
def print_and_assert_on_rebuild_failure(self, out, timeout=3):
"""Print the out value (daos, dmg, etc) and check for rebuild
completion. If not, raise assert.
"""
self.log.info(out)
self.is_rebuild_done(timeout)
self.assert_on_rebuild_failure()
@fail_on(CommandFailure)
def get_pool_version(self):
"""Get the pool version.
Returns:
int: pool_version_value
"""
data = self.dmg_command.pool_query(self.pool.uuid)
return int(data["response"]["version"])
def set_container(self, container):
"""Set the OSA utils container object.
Args:
container (obj) : Container object to be used
within OSA utils.
"""
self.container = container
def simple_exclude_reintegrate_loop(self, rank, loop_time=100):
"""This method performs exclude and reintegration on a rank,
for a certain amount of time.
"""
start_time = 0
finish_time = 0
while int(finish_time - start_time) > loop_time:
start_time = time.time()
output = self.dmg_command.pool_exclude(self.pool.uuid, rank)
self.print_and_assert_on_rebuild_failure(output)
output = self.dmg_command.pool_reintegrate(self.pool.uuid, rank)
self.print_and_assert_on_rebuild_failure(output)
@fail_on(DaosApiError)
def write_single_object(self):
"""Write some data to the existing pool."""
self.pool.connect(2)
csum = self.params.get("enable_checksum", '/run/container/*')
self.container = DaosContainer(self.context)
input_param = self.container.cont_input_values
input_param.enable_chksum = csum
self.container.create(poh=self.pool.pool.handle, con_prop=input_param)
self.container.open()
self.obj = DaosObj(self.context, self.container)
self.obj.create(objcls=1)
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj,
objtype=4)
self.log.info("Writing the Single Dataset")
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0]) * self.record_length)
d_key_value = "dkey {0}".format(dkey)
c_dkey = create_string_buffer(d_key_value)
a_key_value = "akey {0}".format(akey)
c_akey = create_string_buffer(a_key_value)
c_value = create_string_buffer(indata)
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size)
self.obj.close()
self.container.close()
@fail_on(DaosApiError)
def verify_single_object(self):
"""Verify the container data on the existing pool."""
self.pool.connect(2)
self.container.open()
self.obj.open()
self.log.info("Single Dataset Verification -- Started")
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0]) * self.record_length)
c_dkey = create_string_buffer("dkey {0}".format(dkey))
c_akey = create_string_buffer("akey {0}".format(akey))
val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1)
if indata != (repr(val.value)[1:-1]):
self.d_log.error("ERROR:Data mismatch for "
"dkey = {0}, "
"akey = {1}".format(
"dkey {0}".format(dkey),
"akey {0}".format(akey)))
self.fail(
"ERROR: Data mismatch for dkey = {0}, akey={1}".format(
"dkey {0}".format(dkey), "akey {0}".format(akey)))
self.obj.close()
self.container.close()
def prepare_cont_ior_write_read(self, oclass, flags):
"""This method prepares the containers for
IOR write and read invocations.
To enable aggregation:
- Create two containers and read always from
first container
Normal usage (use only a single container):
- Create a single container and use the same.
Args:
oclass (str): IOR object class
flags (str): IOR flags
"""
self.log.info(self.pool_cont_dict)
# If pool is not in the dictionary,
# initialize its container list to None
# {poolA : [None, None], [None, None]}
if self.pool not in self.pool_cont_dict:
self.pool_cont_dict[self.pool] = [None] * 4
# Create container if the pool doesn't have one.
# Otherwise, use the existing container in the pool.
# pool_cont_dict {pool A: [containerA, Updated,
# containerB, Updated],
# pool B : containerA, Updated,
# containerB, None]}
if self.pool_cont_dict[self.pool][0] is None:
self.add_container(self.pool, create=False)
self.set_cont_class_properties(oclass)
if self.test_with_checksum is False:
tmp = self.get_object_replica_value(oclass)
rf_value = "rf:{}".format(tmp - 1)
self.update_cont_properties(rf_value)
self.container.create()
self.pool_cont_dict[self.pool][0] = self.container
self.pool_cont_dict[self.pool][1] = "Updated"
else:
if ((self.test_during_aggregation is True)
and (self.pool_cont_dict[self.pool][1] == "Updated")
and (self.pool_cont_dict[self.pool][3] is None)
and ("-w" in flags)):
# Write to the second container
self.add_container(self.pool, create=False)
self.set_cont_class_properties(oclass)
if self.test_with_checksum is False:
tmp = self.get_object_replica_value(oclass)
rf_value = "rf:{}".format(tmp - 1)
self.update_cont_properties(rf_value)
self.container.create()
self.pool_cont_dict[self.pool][2] = self.container
self.pool_cont_dict[self.pool][3] = "Updated"
else:
self.container = self.pool_cont_dict[self.pool][0]
def delete_extra_container(self, pool):
"""Delete the extra container in the pool.
Refer prepare_cont_ior_write_read. This method
should be called when OSA tests intend to
enable aggregation.
Args:
pool (object): pool handle
"""
self.pool.set_property("reclaim", "time")
extra_container = self.pool_cont_dict[pool][2]
extra_container.destroy()
self.pool_cont_dict[pool][3] = None
def get_object_replica_value(self, oclass):
""" Get the object replica value for an object class.
Args:
oclass (str): Object Class (eg: RP_2G1,etc)
Returns:
value (int) : Object replica value
"""
value = 0
if "_" in oclass:
replica_list = oclass.split("_")
value = replica_list[1][0]
else:
self.log.info("Wrong Object Class. Cannot split")
return int(value)
def update_cont_properties(self, cont_prop):
"""Update the existing container properties.
Args:
cont_prop (str): Replace existing container properties
with new value
"""
self.container.properties.value = cont_prop
def set_cont_class_properties(self, oclass="S1"):
"""Update the container class to match the IOR object
class. Fix the rf factor based on object replica value.
Also, remove the redundancy factor for S type
object class.
Args:
oclass (str, optional): Container object class to be set.
Defaults to "S1".
"""
self.container.oclass.value = oclass
# Set the container properties properly for S!, S2 class.
# rf should not be set to 1 for S type object class.
x = re.search("^S\\d$", oclass)
prop = self.container.properties.value
if x is not None:
prop = prop.replace("rf:1", "rf:0")
else:
tmp = self.get_object_replica_value(oclass)
rf_value = "rf:{}".format(tmp - 1)
prop = prop.replace("rf:1", rf_value)
self.container.properties.value = prop
def assert_on_exception(self, out_queue=None):
"""Assert on exception while executing an application.
Args:
out_queue (queue): Check whether the queue is
empty. If empty, app (ior, mdtest) didn't encounter error.
"""
if out_queue is None:
out_queue = self.out_queue
if out_queue.empty():
pass
else:
exc = out_queue.get(block=False)
out_queue.put(exc)
raise exc
def cleanup_queue(self, out_queue=None):
"""Cleanup the existing thread queue.
Args:
out_queue (queue): Queue to cleanup.
"""
if out_queue is None:
out_queue = self.out_queue
while not out_queue.empty():
out_queue.get(block=True)
def run_ior_thread(self,
action,
oclass,
test,
single_cont_read=True,
fail_on_warning=True):
"""Start the IOR thread for either writing or
reading data to/from a container.
Args:
action (str): Start the IOR thread with Read or
Write
oclass (str): IOR object class
test (list): IOR test sequence
flags (str): IOR flags
single_cont_read (bool) : Always read from the
1st container.
Defaults to True.
fail_on_warning (bool) : Test terminates
for IOR warnings.
Defaults to True.
"""
self.cleanup_queue()
if action == "Write":
flags = self.ior_w_flags
else:
flags = self.ior_r_flags
# Add a thread for these IOR arguments
process = threading.Thread(target=self.ior_thread,
kwargs={
"pool": self.pool,
"oclass": oclass,
"test": test,
"flags": flags,
"single_cont_read": single_cont_read,
"fail_on_warning": fail_on_warning
})
# Launch the IOR thread
process.start()
# Wait for the thread to finish
try:
process.join()
except CommandFailure as err_msg:
self.out_queue.put(err_msg)
self.assert_on_exception()
def ior_thread(self,
pool,
oclass,
test,
flags,
single_cont_read=True,
fail_on_warning=True):
"""Start an IOR thread.
Args:
pool (object): pool handle
oclass (str): IOR object class, container class.
test (list): IOR test sequence
flags (str): IOR flags
single_cont_read (bool) : Always read from the
1st container.
Defaults to True.
fail_on_warning (bool) : Test terminates
for IOR warnings.
Defaults to True.
"""
self.cleanup_queue()
self.pool = pool
self.ior_cmd.get_params(self)
self.ior_cmd.set_daos_params(self.server_group, self.pool)
self.ior_cmd.dfs_oclass.update(oclass)
self.ior_cmd.dfs_dir_oclass.update(oclass)
if single_cont_read is True:
# Prepare the containers created and use in a specific
# way defined in prepare_cont_ior_write.
self.prepare_cont_ior_write_read(oclass, flags)
elif single_cont_read is False and self.container is not None:
# Here self.container is having actual value. Just use it.
self.log.info(self.container)
else:
self.fail("Not supported option on ior_thread")
try:
job_manager = self.get_ior_job_manager_command()
except CommandFailure as err_msg:
self.out_queue.put(err_msg)
self.assert_on_exception()
job_manager.job.dfs_cont.update(self.container.uuid)
self.ior_cmd.transfer_size.update(test[2])
self.ior_cmd.block_size.update(test[3])
self.ior_cmd.flags.update(flags)
try:
self.run_ior_with_pool(create_pool=False,
create_cont=False,
fail_on_warning=fail_on_warning)
except CommandFailure as err_msg:
self.out_queue.put(err_msg)
self.assert_on_exception()
def run_mdtest_thread(self):
"""Start mdtest thread and wait until thread completes.
"""
# Create container only
self.mdtest_cmd.dfs_destroy = False
if self.container is None:
self.add_container(self.pool, create=False)
self.set_cont_class_properties(self.mdtest_cmd.dfs_oclass)
if self.test_with_checksum is False:
tmp = self.get_object_replica_value(self.mdtest_cmd.dfs_oclass)
rf_value = "rf:{}".format(tmp - 1)
self.update_cont_properties(rf_value)
self.container.create()
job_manager = self.get_mdtest_job_manager_command(self.manager)
job_manager.job.dfs_cont.update(self.container.uuid)
# Add a thread for these IOR arguments
process = threading.Thread(target=self.execute_mdtest)
# Launch the MDtest thread
process.start()
# Wait for the thread to finish
try:
process.join()
except CommandFailure as err_msg:
self.out_queue.put(err_msg)
self.assert_on_exception()
class PunchTest(TestWithServers):
"""
Simple test to verify the 3 different punch calls.
:avocado: recursive
"""
def setUp(self):
try:
super(PunchTest, self).setUp()
# parameters used in pool create
createmode = self.params.get("mode", '/run/pool/createmode/')
createsetid = self.params.get("setname", '/run/pool/createset/')
createsize = self.params.get("size", '/run/pool/createsize/')
createuid = os.geteuid()
creategid = os.getegid()
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
self.pool.connect(1 << 1)
# create a container
self.container = DaosContainer(self.context)
self.container.create(self.pool.handle)
# now open it
self.container.open()
except DaosApiError as excpn:
print(excpn)
print(traceback.format_exc())
self.fail("Test failed during setup.\n")
def test_dkey_punch(self):
"""
The most basic test of the dkey punch function.
:avocado: tags=all,object,pr,small,dkeypunch
"""
try:
# create an object and write some data into it
thedata = "a string that I want to stuff into an object"
dkey = "this is the dkey"
akey = "this is the akey"
obj, txn = self.container.write_an_obj(thedata,
len(thedata) + 1,
dkey,
akey,
obj_cls=1)
# read the data back and make sure its correct
thedata2 = self.container.read_an_obj(
len(thedata) + 1, dkey, akey, obj, txn)
if thedata != thedata2.value:
print("data I wrote:" + thedata)
print("data I read back" + thedata2.value)
self.fail("Wrote data, read it back, didn't match\n")
# now punch this data, should fail, can't punch committed data
obj.punch_dkeys(txn, [dkey])
# expecting punch of commit data above to fail
self.fail("Punch should have failed but it didn't.\n")
# expecting an exception so do nothing
except DaosApiError as dummy_e:
pass
try:
# now punch this data
obj.punch_dkeys(0, [dkey])
# this one should work so error if exception occurs
except DaosApiError as dummy_e:
self.fail("Punch should have worked.\n")
# there are a bunch of other cases to test here,
# --test punching the same updating and punching the same data in
# the same tx, should fail
# --test non updated data in an open tx, should work
def test_akey_punch(self):
"""
The most basic test of the akey punch function.
:avocado: tags=all,object,pr,small,akeypunch
"""
try:
# create an object and write some data into it
dkey = "this is the dkey"
data1 = [("this is akey 1", "this is data value 1"),
("this is akey 2", "this is data value 2"),
("this is akey 3", "this is data value 3")]
obj, txn = self.container.write_multi_akeys(dkey, data1, obj_cls=1)
# read back the 1st epoch's data and check 1 value just to make sure
# everything is on the up and up
readbuf = [(data1[0][0], len(data1[0][1]) + 1),
(data1[1][0], len(data1[1][1]) + 1),
(data1[2][0], len(data1[2][1]) + 1)]
retrieved_data = self.container.read_multi_akeys(
dkey, readbuf, obj, txn)
if retrieved_data[data1[1][0]] != data1[1][1]:
print("middle akey: {}".format(retrieved_data[data1[1][0]]))
self.fail("data retrieval failure")
# now punch one akey from this data
obj.punch_akeys(txn, dkey, [data1[1][0]])
# expecting punch of commit data above to fail
self.fail("Punch should have failed but it didn't.\n")
# expecting an exception so do nothing
except DaosApiError as excep:
print(excep)
try:
# now punch the object without a tx
obj.punch_akeys(0, dkey, [data1[1][0]])
# expecting it to work this time so error
except DaosApiError as excep:
self.fail("Punch should have worked: {}\n".format(excep))
def test_obj_punch(self):
"""
The most basic test of the object punch function. Really similar
to above except the whole object is deleted.
:avocado: tags=all,object,pr,small,objpunch
"""
try:
# create an object and write some data into it
thedata = "a string that I want to stuff into an object"
dkey = "this is the dkey"
akey = "this is the akey"
obj, txn = self.container.write_an_obj(thedata,
len(thedata) + 1,
dkey,
akey,
obj_cls=1)
# read the data back and make sure its correct
thedata2 = self.container.read_an_obj(
len(thedata) + 1, dkey, akey, obj, txn)
if thedata != thedata2.value:
print("data I wrote:" + thedata)
print("data I read back" + thedata2.value)
self.fail("Wrote data, read it back, didn't match\n")
# now punch the object, commited so not expecting it to work
obj.punch(txn)
# expecting punch of commit data above to fail
self.fail("Punch should have failed but it didn't.\n")
# expecting an exception so do nothing
except DaosApiError as excep:
print(excep)
try:
obj.punch(0)
# expecting it to work without a tx
except DaosApiError as excep:
print(excep)
self.fail("Punch should have worked.\n")
class OSAUtils(MdtestBase, IorTestBase):
# pylint: disable=too-many-ancestors
"""
Test Class Description: This test runs
daos_server offline drain test cases.
:avocado: recursive
"""
def setUp(self):
"""Set up for test case."""
super().setUp()
self.pool_cont_dict = {}
self.container = None
self.obj = None
self.ioreq = None
self.dmg_command = self.get_dmg_command()
self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*',
default=[0])[0]
self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*',
default=[0])[0]
self.record_length = self.params.get("length", '/run/record/*',
default=[0])[0]
self.ior_w_flags = self.params.get("write_flags", '/run/ior/iorflags/*',
default="")
self.ior_r_flags = self.params.get("read_flags", '/run/ior/iorflags/*')
self.server_count = len(self.hostlist_servers)
self.engine_count = self.server_managers[0].get_config_value(
"engines_per_host")
self.out_queue = queue.Queue()
self.dmg_command.exit_status_exception = False
self.test_during_aggregation = False
self.test_during_rebuild = False
self.test_with_checksum = True
# By default, test_with_rf is set to False.
# It is up to individual test to enable it.
self.test_with_rf = False
self.test_with_blank_node = False
self.test_with_snapshot = False
@fail_on(CommandFailure)
def get_pool_leader(self):
"""Get the pool leader.
Returns:
int: pool leader value
"""
data = self.dmg_command.pool_query(self.pool.uuid)
return int(data["response"]["leader"])
@fail_on(CommandFailure)
def get_rebuild_status(self):
"""Get the rebuild status.
Returns:
str: rebuild status
"""
data = self.dmg_command.pool_query(self.pool.uuid)
return data["response"]["rebuild"]["status"]
@fail_on(CommandFailure)
def get_rebuild_state(self):
"""Get the rebuild state.
Returns:
str: rebuild state
"""
data = self.dmg_command.pool_query(self.pool.uuid)
return data["response"]["rebuild"]["state"]
@fail_on(CommandFailure)
def is_rebuild_done(self, time_interval,
wait_for_rebuild_to_complete=False):
"""Rebuild is completed/done.
Args:
time_interval: Wait interval between checks
wait_for_rebuild_to_complete: Rebuild completed
(Default: False)
"""
self.pool.wait_for_rebuild(wait_for_rebuild_to_complete,
interval=time_interval)
@fail_on(CommandFailure)
def assert_on_rebuild_failure(self):
"""If the rebuild is not successful,
raise assert.
"""
rebuild_status = self.get_rebuild_status()
self.log.info("Rebuild Status: %s", rebuild_status)
rebuild_failed_string = ["failed", "scanning", "aborted", "busy"]
self.assertTrue(rebuild_status not in rebuild_failed_string,
"Rebuild failed")
@fail_on(CommandFailure)
def print_and_assert_on_rebuild_failure(self, out, timeout=3):
"""Print the out value (daos, dmg, etc) and check for rebuild
completion. If not, raise assert.
"""
self.log.info(out)
self.is_rebuild_done(timeout)
self.assert_on_rebuild_failure()
@fail_on(CommandFailure)
def get_pool_version(self):
"""Get the pool version.
Returns:
int: pool_version_value
"""
data = self.dmg_command.pool_query(self.pool.uuid)
return int(data["response"]["version"])
@fail_on(CommandFailure)
def get_ipaddr_for_rank(self, rank=None):
"""Obtain the IPAddress and port number for a
particular server rank.
Args:
rank (int): daos_engine rank. Defaults to None.
Returns:
ip_addr (str) : IPAddress for the rank.
port_num (str) : Port number for the rank.
"""
output = self.dmg_command.system_query()
members_length = self.server_count * self.engine_count
for i in range(0, members_length):
if rank == int(output["response"]["members"][i]["rank"]):
temp = output["response"]["members"][i]["addr"]
ip_addr = temp.split(":")
temp = output["response"]["members"][i]["fabric_uri"]
port_num = temp.split(":")
return ip_addr[0], port_num[2]
return None, None
@fail_on(CommandFailure)
def remove_pool_dir(self, ip_addr=None, port_num=None):
"""Remove the /mnt/daos[x]/<pool_uuid>/vos-* directory
Args:
ip_addr (str): IP address of the daos server.
Defaults to None.
port_number (str) : Port number the daos server.
"""
# Create the expected port list
# expected_ports = [port0] - Single engine/server
# expected_ports = [port0, port1] - Two engine/server
expected_ports = [engine_param.get_value("fabric_iface_port")
for engine_param in self.server_managers[-1].
manager.job.yaml.engine_params]
self.log.info("Expected ports : %s", expected_ports)
if ip_addr is None or port_num is None:
self.log.info("ip_addr : %s port_number: %s", ip_addr, port_num)
self.fail("No IP Address or Port number provided")
else:
if self.engine_count == 1:
self.log.info("Single Engine per Server")
cmd = "/usr/bin/ssh {} -oStrictHostKeyChecking=no \
sudo rm -rf /mnt/daos/{}/vos-*". \
format(ip_addr, self.pool.uuid)
elif self.engine_count == 2:
if port_num == str(expected_ports[0]):
port_val = 0
elif port_num == str(expected_ports[1]):
port_val = 1
else:
self.log.info("port_number: %s", port_num)
self.fail("Invalid port number")
cmd = "/usr/bin/ssh {} -oStrictHostKeyChecking=no \
sudo rm -rf /mnt/daos{}/{}/vos-*". \
format(ip_addr, port_val, self.pool.uuid)
else:
self.fail("Not supported engine per server configuration")
run_command(cmd)
def set_container(self, container):
"""Set the OSA utils container object.
Args:
container (obj) : Container object to be used
within OSA utils.
"""
self.container = container
def simple_osa_reintegrate_loop(self, rank, action="exclude",
loop_time=100):
"""This method performs exclude or drain and
reintegration on a rank for a certain amount of time.
Args:
rank (int): daos server rank.
action (str) : "exclude" or "drain".
Defaults to "exclude"
loop_time: Total time to perform drain/reintegrate
operation in a loop. (Default : 100 secs)
"""
start_time = 0
finish_time = 0
start_time = time.time()
while int(finish_time - start_time) < loop_time:
if action == "exclude":
output = self.dmg_command.pool_exclude(self.pool.uuid,
rank)
else:
output = self.dmg_command.pool_drain(self.pool.uuid,
rank)
self.print_and_assert_on_rebuild_failure(output)
output = self.dmg_command.pool_reintegrate(self.pool.uuid,
rank)
self.print_and_assert_on_rebuild_failure(output)
finish_time = time.time()
@fail_on(DaosApiError)
def write_single_object(self):
"""Write some data to the existing pool."""
self.pool.connect(2)
csum = self.params.get("enable_checksum", '/run/container/*')
self.container = DaosContainer(self.context)
input_param = self.container.cont_input_values
input_param.enable_chksum = csum
self.container.create(poh=self.pool.pool.handle,
con_prop=input_param)
self.container.open()
self.obj = DaosObj(self.context, self.container)
self.obj.create(objcls=1)
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj, objtype=4)
self.log.info("Writing the Single Dataset")
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0])
* self.record_length)
d_key_value = "dkey {0}".format(dkey)
c_dkey = create_string_buffer(d_key_value)
a_key_value = "akey {0}".format(akey)
c_akey = create_string_buffer(a_key_value)
c_value = create_string_buffer(indata)
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size)
self.obj.close()
self.container.close()
@fail_on(DaosApiError)
def verify_single_object(self):
"""Verify the container data on the existing pool."""
self.pool.connect(2)
self.container.open()
self.obj.open()
self.log.info("Single Dataset Verification -- Started")
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0]) *
self.record_length)
c_dkey = create_string_buffer("dkey {0}".format(dkey))
c_akey = create_string_buffer("akey {0}".format(akey))
val = self.ioreq.single_fetch(c_dkey,
c_akey,
len(indata)+1)
if indata != (repr(val.value)[1:-1]):
self.d_log.error("ERROR:Data mismatch for "
"dkey = {0}, "
"akey = {1}".format(
"dkey {0}".format(dkey),
"akey {0}".format(akey)))
self.fail("ERROR: Data mismatch for dkey = {0}, akey={1}"
.format("dkey {0}".format(dkey),
"akey {0}".format(akey)))
self.obj.close()
self.container.close()
def prepare_cont_ior_write_read(self, oclass, flags):
"""This method prepares the containers for
IOR write and read invocations.
To enable aggregation:
- Create two containers and read always from
first container
Normal usage (use only a single container):
- Create a single container and use the same.
Args:
oclass (str): IOR object class
flags (str): IOR flags
"""
self.log.info(self.pool_cont_dict)
# If pool is not in the dictionary,
# initialize its container list to None
# {poolA : [None, None], [None, None]}
if self.pool not in self.pool_cont_dict:
self.pool_cont_dict[self.pool] = [None] * 4
# Create container if the pool doesn't have one.
# Otherwise, use the existing container in the pool.
# pool_cont_dict {pool A: [containerA, Updated,
# containerB, Updated],
# pool B : containerA, Updated,
# containerB, None]}
if self.pool_cont_dict[self.pool][0] is None:
self.add_container(self.pool, create=False)
self.set_cont_class_properties(oclass)
if self.test_with_checksum is False:
tmp = self.get_object_replica_value(oclass)
rf_value = "rf:{}".format(tmp - 1)
self.update_cont_properties(rf_value)
self.container.create()
self.pool_cont_dict[self.pool][0] = self.container
self.pool_cont_dict[self.pool][1] = "Updated"
else:
if ((self.test_during_aggregation is True) and
(self.pool_cont_dict[self.pool][1] == "Updated") and
(self.pool_cont_dict[self.pool][3] is None) and
("-w" in flags)):
# Write to the second container
self.add_container(self.pool, create=False)
self.set_cont_class_properties(oclass)
if self.test_with_checksum is False:
tmp = self.get_object_replica_value(oclass)
rf_value = "rf:{}".format(tmp - 1)
self.update_cont_properties(rf_value)
self.container.create()
self.pool_cont_dict[self.pool][2] = self.container
self.pool_cont_dict[self.pool][3] = "Updated"
else:
self.container = self.pool_cont_dict[self.pool][0]
def delete_extra_container(self, pool):
"""Delete the extra container in the pool.
Refer prepare_cont_ior_write_read. This method
should be called when OSA tests intend to
enable aggregation.
Args:
pool (object): pool handle
"""
self.pool.set_property("reclaim", "time")
extra_container = self.pool_cont_dict[pool][2]
extra_container.destroy()
self.pool_cont_dict[pool][3] = None
def get_object_replica_value(self, oclass):
""" Get the object replica value for an object class.
Args:
oclass (str): Object Class (eg: RP_2G1,etc)
Returns:
value (int) : Object replica value
"""
value = 0
if "_" in oclass:
replica_list = oclass.split("_")
value = replica_list[1][0]
else:
self.log.info("Wrong Object Class. Cannot split")
return int(value)
def update_cont_properties(self, cont_prop):
"""Update the existing container properties.
Args:
cont_prop (str): Replace existing container properties
with new value
"""
self.container.properties.value = cont_prop
def set_cont_class_properties(self, oclass="S1"):
"""Update the container class to match the IOR/Mdtest object
class. Fix the rf factor based on object replica value.
Also, remove the redundancy factor for S type
object class.
Args:
oclass (str, optional): Container object class to be set.
Defaults to "S1".
"""
self.container.oclass.value = oclass
# Set the container properties properly for S!, S2 class.
# rf should not be set to 1 for S type object class.
x = re.search("^S\\d$", oclass)
prop = self.container.properties.value
if x is not None:
prop = prop.replace("rf:1", "rf:0")
else:
tmp = self.get_object_replica_value(oclass)
rf_value = "rf:{}".format(tmp - 1)
prop = prop.replace("rf:1", rf_value)
self.container.properties.value = prop
# Over-write oclass settings if using redundancy factor
# and self.test_with_rf is True.
# This has to be done so that container created doesn't
# use the object class.
if self.test_with_rf is True and \
"rf" in self.container.properties.value:
self.log.info(
"Detected container redundancy factor: %s",
self.container.properties.value)
self.ior_cmd.dfs_oclass.update(None, "ior.dfs_oclass")
self.ior_cmd.dfs_dir_oclass.update(None, "ior.dfs_dir_oclass")
self.container.oclass.update(None)
def assert_on_exception(self, out_queue=None):
"""Assert on exception while executing an application.
Args:
out_queue (queue): Check whether the queue is
empty. If empty, app (ior, mdtest) didn't encounter error.
"""
if out_queue is None:
out_queue = self.out_queue
if out_queue.empty():
pass
else:
exc = out_queue.get(block=False)
out_queue.put(exc)
raise CommandFailure(exc)
def cleanup_queue(self, out_queue=None):
"""Cleanup the existing thread queue.
Args:
out_queue (queue): Queue to cleanup.
"""
if out_queue is None:
out_queue = self.out_queue
while not out_queue.empty():
out_queue.get(block=True)
def run_ior_thread(self, action, oclass, test, single_cont_read=True,
fail_on_warning=True, pool=None):
"""Start the IOR thread for either writing or
reading data to/from a container.
Args:
action (str): Start the IOR thread with Read or
Write
oclass (str): IOR object class
test (list): IOR test sequence
flags (str): IOR flags
single_cont_read (bool) : Always read from the
1st container.
Defaults to True.
fail_on_warning (bool) : Test terminates
for IOR warnings.
Defaults to True.
pool (TestPool): Pool to run ior on. Defaults to None.
"""
# Intermediate (between correct and hack) implementation for allowing a
# pool to be passed in. Needs to be fixed by making the pool argument
# required.
if pool is None:
pool = self.pool
self.cleanup_queue()
if action == "Write":
flags = self.ior_w_flags
else:
flags = self.ior_r_flags
# Add a thread for these IOR arguments
process = threading.Thread(target=self.ior_thread,
kwargs={"pool": pool,
"oclass": oclass,
"test": test,
"flags": flags,
"single_cont_read":
single_cont_read,
"fail_on_warning":
fail_on_warning})
# Launch the IOR thread
process.start()
# Wait for the thread to finish
process.join()
if not self.out_queue.empty():
self.assert_on_exception()
def ior_thread(self, pool, oclass, test, flags,
single_cont_read=True,
fail_on_warning=True):
"""Start an IOR thread.
Args:
pool (object): pool handle
oclass (str): IOR object class, container class.
test (list): IOR test sequence
flags (str): IOR flags
single_cont_read (bool) : Always read from the
1st container.
Defaults to True.
fail_on_warning (bool) : Test terminates
for IOR warnings.
Defaults to True.
"""
self.cleanup_queue()
self.pool = pool
self.ior_cmd.get_params(self)
self.ior_cmd.set_daos_params(self.server_group, self.pool)
self.log.info("Redundancy Factor : %s", self.test_with_rf)
self.ior_cmd.dfs_oclass.update(oclass)
self.ior_cmd.dfs_dir_oclass.update(oclass)
if single_cont_read is True:
# Prepare the containers created and use in a specific
# way defined in prepare_cont_ior_write.
self.prepare_cont_ior_write_read(oclass, flags)
elif single_cont_read is False and self.container is not None:
# Here self.container is having actual value. Just use it.
self.log.info(self.container)
else:
self.fail("Not supported option on ior_thread")
try:
job_manager = self.get_ior_job_manager_command()
except CommandFailure as err_msg:
self.out_queue.put(err_msg)
self.assert_on_exception()
job_manager.job.dfs_cont.update(self.container.uuid)
self.ior_cmd.transfer_size.update(test[2])
self.ior_cmd.block_size.update(test[3])
self.ior_cmd.flags.update(flags)
# Update oclass settings if using redundancy factor
# and self.test_with_rf is True.
if self.test_with_rf is True and \
"rf" in self.container.properties.value:
self.log.info(
"Detected container redundancy factor: %s",
self.container.properties.value)
self.ior_cmd.dfs_oclass.update(None, "ior.dfs_oclass")
self.ior_cmd.dfs_dir_oclass.update(None, "ior.dfs_dir_oclass")
self.run_ior_with_pool(create_pool=False, create_cont=False,
fail_on_warning=fail_on_warning,
out_queue=self.out_queue)
if not self.out_queue.empty():
self.assert_on_exception()
def run_mdtest_thread(self, oclass="RP_2G1"):
"""Start mdtest thread and wait until thread completes.
Args:
oclass (str): IOR object class, container class.
"""
# Create container only
self.mdtest_cmd.dfs_destroy = False
create_container = 0
if self.container is None:
self.add_container(self.pool, create=False)
create_container = 1
self.mdtest_cmd.dfs_oclass.update(oclass)
self.set_cont_class_properties(oclass)
if self.test_with_checksum is False:
tmp = self.get_object_replica_value(oclass)
rf_value = "rf:{}".format(tmp - 1)
self.update_cont_properties(rf_value)
if create_container == 1:
self.container.create()
job_manager = self.get_mdtest_job_manager_command(self.manager)
job_manager.job.dfs_cont.update(self.container.uuid)
# Add a thread for these IOR arguments
process = threading.Thread(target=self.execute_mdtest)
# Launch the MDtest thread
process.start()
# Wait for the thread to finish
process.join()
if not self.out_queue.empty():
self.assert_on_exception()
class GlobalHandle(TestWithServers):
"""
This class contains tests to verify the ability to share container
handles amoung processes.
:avocado: recursive
"""
def tearDown(self):
try:
super(GlobalHandle, self).tearDown()
finally:
# really make sure everything is gone
check_for_pool.cleanup_pools(self.hostlist_servers)
@fail_on(DaosApiError)
def check_handle(self, pool_glob_handle, uuidstr, cont_glob_handle, rank):
"""
This gets run in a child process and verifyes the global
handles can be turned into local handles in another process.
"""
# setup the pool and connect using global handle
pool = DaosPool(self.context)
pool.uuid = uuidstr
pool.set_svc(rank)
pool.group = "daos_server"
buf = ctypes.cast(
pool_glob_handle.iov_buf,
ctypes.POINTER(ctypes.c_byte * pool_glob_handle.iov_buf_len))
buf2 = bytearray()
buf2.extend(buf.contents)
pool_handle = pool.global2local(self.context, pool_glob_handle.iov_len,
pool_glob_handle.iov_buf_len, buf2)
# perform an operation that will use the new handle, if it
# doesn't throw an exception, then all is well.
pool.pool_query()
# setup the container and then connect using the global handle
container = DaosContainer(self.context)
container.poh = pool_handle
buf = ctypes.cast(
cont_glob_handle.iov_buf,
ctypes.POINTER(ctypes.c_byte * cont_glob_handle.iov_buf_len))
buf2 = bytearray()
buf2.extend(buf.contents)
dummy_cont_handle = container.global2local(
self.context, cont_glob_handle.iov_len,
cont_glob_handle.iov_buf_len, buf2)
# just try one thing to make sure handle is good
container.query()
def test_global_handle(self):
"""
Test ID: DAO
Test Description: Use a pool handle in another process.
:avocado: tags=all,container,tiny,pr,conthandle
"""
try:
# use the uid/gid of the user running the test, these should
# be perfectly valid
createuid = os.geteuid()
creategid = os.getegid()
# parameters used in pool create that are in yaml
createmode = self.params.get("mode", '/run/testparams/createmode/')
createsetid = self.params.get("setname",
'/run/testparams/createset/')
createsize = self.params.get("size", '/run/testparams/createsize/')
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.pool.create(createmode, createuid, creategid, createsize,
createsetid, None)
self.pool.connect(1 << 1)
# create a pool global handle
iov_len, buf_len, buf = self.pool.local2global()
buftype = ctypes.c_byte * buf_len
c_buf = buftype.from_buffer(buf)
sct_pool_handle = (sharedctypes.RawValue(
IOV, ctypes.cast(c_buf, ctypes.c_void_p), buf_len, iov_len))
# create a container
self.container = DaosContainer(self.context)
self.container.create(self.pool.handle)
self.container.open()
# create a container global handle
iov_len, buf_len, buf = self.container.local2global()
buftype = ctypes.c_byte * buf_len
c_buf = buftype.from_buffer(buf)
sct_cont_handle = (sharedctypes.RawValue(
IOV, ctypes.cast(c_buf, ctypes.c_void_p), buf_len, iov_len))
sct_pool_uuid = sharedctypes.RawArray(ctypes.c_byte,
self.pool.uuid)
# this should work in the future but need on-line server addition
#arg_list = (
#p = Process(target=check_handle, args=arg_list)
#p.start()
#p.join()
# for now verifying global handle in the same process which is not
# the intended use case
self.check_handle(sct_pool_handle, sct_pool_uuid, sct_cont_handle,
0)
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
self.fail("Expecting to pass but test has failed.\n")
class ObjectDataValidation(TestWithServers):
"""
Test Class Description:
Tests that create Different length records,
Disconnect the pool/container and reconnect,
validate the data after reconnect.
:avocado: recursive
"""
# pylint: disable=too-many-instance-attributes
def setUp(self):
super(ObjectDataValidation, self).setUp()
self.obj = None
self.ioreq = None
self.no_of_dkeys = None
self.no_of_akeys = None
self.array_size = None
self.record_length = None
self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0]
self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0]
self.array_size = self.params.get("size", '/array_size/')
self.record_length = self.params.get("length", '/run/record/*')
self.prepare_pool()
self.container = DaosContainer(self.context)
self.container.create(self.pool.pool.handle)
self.container.open()
self.obj = DaosObj(self.context, self.container)
self.obj.create(objcls=1)
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj,
objtype=4)
def reconnect(self):
'''
Function to reconnect the pool/container and reopen the Object
for read verification.
'''
#Close the Obj/Container, Disconnect the Pool.
self.obj.close()
self.container.close()
self.pool.disconnect()
time.sleep(5)
#Connect Pool, Open Container and Object
self.pool.connect(2)
self.container.open()
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj,
objtype=4)
@avocado.fail_on(DaosApiError)
def test_invalid_tx_commit_close(self):
"""
Test ID:
(1)DAOS-1346: Verify commit tx bad parameter behavior.
(2)DAOS-1343: Verify tx_close bad parameter behavior.
(3)DAOS-1342: Verify tx_close through daos_api.
(4)DAOS-1338: Add and verify tx_abort through daos_api.
(5)DAOS-1339: Verify tx_abort bad parameter behavior.
Test Description:
Write Avocado Test to verify commit tx and close tx
bad parameter behavior.
:avocado: tags=all,object,full_regression,small,invalid_tx
"""
self.d_log.info("==Writing the Single Dataset for negative test...")
record_index = 0
expected_error = "RC: -1002"
dkey = 0
akey = 0
indata = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
c_value = ctypes.create_string_buffer(indata)
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
try:
new_transaction = self.container.get_new_tx()
except DaosApiError as excep:
#initial container get_new_tx failed, skip rest of the test
self.fail("##container get_new_tx failed: {}".format(excep))
invalid_transaction = new_transaction + random.randint(1000, 383838)
self.log.info("==new_transaction= %s", new_transaction)
self.log.info("==invalid_transaction= %s", invalid_transaction)
self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
new_transaction)
try:
self.container.commit_tx(invalid_transaction)
self.fail("##(1.1)Container.commit_tx passing with invalid handle")
except DaosApiError as excep:
self.log.info(str(excep))
self.log.info(
"==(1)Expecting failure: invalid Container.commit_tx.")
if expected_error not in str(excep):
self.fail(
"##(1.2)Expecting error RC: -1002, but got {}.".format(
str(excep)))
try:
self.container.close_tx(invalid_transaction)
self.fail("##(2.1)Container.close_tx passing with invalid handle")
except DaosApiError as excep:
self.log.info(str(excep))
self.log.info(
"==(2)Expecting failure: invalid Container.commit_tx.")
if expected_error not in str(excep):
self.fail(
"##(2.2)Expecting error RC: -1002, but got {}.".format(
str(excep)))
try:
self.container.close_tx(new_transaction)
self.log.info("==(3)container.close_tx test passed.")
except DaosApiError as excep:
self.log.info(str(excep))
self.fail("##(3)Failed on close_tx.")
try:
self.container.abort_tx(invalid_transaction)
self.fail("##(4.1)Container.abort_tx passing with invalid handle")
except DaosApiError as excep:
self.log.info(str(excep))
self.log.info(
"==(4)Expecting failure: invalid Container.abort_tx.")
if expected_error not in str(excep):
self.fail(
"##(4.2)Expecting error RC: -1002, but got {}.".format(
str(excep)))
#Try to abort the transaction which already closed.
try:
self.container.abort_tx(new_transaction)
self.fail("##(5.1)Container.abort_tx passing with a closed handle")
except DaosApiError as excep:
self.log.info(str(excep))
self.log.info(
"==(5)Expecting failure: Container.abort_tx closed handle.")
if expected_error not in str(excep):
self.fail(
"##(5.2)Expecting error RC: -1002, but got {}.".format(
str(excep)))
#open another transaction for abort test
try:
new_transaction2 = self.container.get_new_tx()
except DaosApiError as excep:
self.fail("##(6.1)container get_new_tx failed: {}".format(excep))
self.log.info("==new_transaction2= %s", new_transaction2)
self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
new_transaction2)
try:
self.container.abort_tx(new_transaction2)
self.log.info("==(6)container.abort_tx test passed.")
except DaosApiError as excep:
self.log.info(str(excep))
self.fail("##(6.2)Failed on abort_tx.")
self.container.close_tx(new_transaction2)
@avocado.fail_on(DaosApiError)
def test_single_object_validation(self):
"""
Test ID: DAOS-707
Test Description: Write Avocado Test to verify single data after
pool/container disconnect/reconnect.
:avocado: tags=all,object,full_regression,small,single_object
"""
self.d_log.info("Writing the Single Dataset")
record_index = 0
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
c_value = ctypes.create_string_buffer(indata)
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size)
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
self.reconnect()
self.d_log.info("Single Dataset Verification -- Started")
record_index = 0
transaction_index = 0
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1)
if indata != (repr(val.value)[1:-1]):
self.d_log.error("ERROR:Data mismatch for "
"dkey = {0}, "
"akey = {1}".format(
"dkey {0}".format(dkey),
"akey {0}".format(akey)))
self.fail(
"ERROR: Data mismatch for dkey = {0}, akey={1}".format(
"dkey {0}".format(dkey), "akey {0}".format(akey)))
transaction_index = transaction_index + 1
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
@avocado.fail_on(DaosApiError)
def test_array_object_validation(self):
"""
Test ID: DAOS-707
Test Description: Write Avocado Test to verify Array data after
pool/container disconnect/reconnect.
:avocado: tags=all,object,full_regression,small,array_object
"""
self.d_log.info("Writing the Array Dataset")
record_index = 0
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
c_values = []
value = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
for item in range(self.array_size):
c_values.append(
(ctypes.create_string_buffer(value), len(value) + 1))
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
self.ioreq.insert_array(c_dkey, c_akey, c_values)
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
self.reconnect()
self.d_log.info("Array Dataset Verification -- Started")
record_index = 0
transaction_index = 0
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = []
value = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
for item in range(self.array_size):
indata.append(value)
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
c_rec_count = ctypes.c_uint(len(indata))
c_rec_size = ctypes.c_size_t(len(indata[0]) + 1)
outdata = self.ioreq.fetch_array(c_dkey, c_akey, c_rec_count,
c_rec_size)
for item in enumerate(indata):
if indata[item[0]] != outdata[item[0]][:-1]:
self.d_log.error("ERROR:Data mismatch for "
"dkey = {0}, "
"akey = {1}".format(
"dkey {0}".format(dkey),
"akey {0}".format(akey)))
self.fail(
"ERROR:Data mismatch for dkey = {0}, akey={1}".
format("dkey {0}".format(dkey),
"akey {0}".format(akey)))
transaction_index = transaction_index + 1
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
class Snapshot(TestWithServers):
"""
Epic: DAOS-2249 Create system level tests that cover basic snapshot
functionality.
Testcase:
DAOS-1370 Basic snapshot test
DAOS-1386 Test container SnapShot information
DAOS-1371 Test list snapshots
DAOS-1395 Test snapshot destroy
DAOS-1402 Test creating multiple snapshots
Test Class Description:
Start DAOS servers, set up the pool and container for the above
snapshot Epic and Testcases, including snapshot basic, container
information, list, creation and destroy.
:avocado: recursive
"""
def setUp(self):
"""
set up method
"""
super(Snapshot, self).setUp()
self.log.info("==In setUp, self.context= %s", self.context)
# initialize a python pool object then create the underlying
# daos storage and connect to it
self.prepare_pool()
try:
# create a container
self.container = DaosContainer(self.context)
self.container.create(self.pool.pool.handle)
except DaosApiError as error:
self.log.info("Error detected in DAOS pool container setup: %s",
str(error))
self.log.info(traceback.format_exc())
self.fail("##Test failed on setUp, before snapshot taken")
# now open it
self.container.open()
# do a query and compare the UUID returned from create with
# that returned by query
self.container.query()
if self.container.get_uuid_str() != c_uuid_to_str(
self.container.info.ci_uuid):
self.fail("##Container UUID did not match the one in info.")
def display_snapshot(self, snapshot):
"""
To display the snapshot information.
Args:
snapshot: snapshot handle to be displayed.
Return:
none.
"""
self.log.info("==display_snapshot================")
self.log.info("snapshot= %s", snapshot)
self.log.info("snapshot.context= %s", snapshot.context)
self.log.info("snapshot.context.libdaos= %s", snapshot.context.libdaos)
self.log.info("snapshot.context.libtest= %s", snapshot.context.libtest)
self.log.info("snapshot.context.ftable= %s", snapshot.context.ftable)
self.log.info("snapshot.context.ftable[list-attr]= %s",
snapshot.context.ftable["list-attr"])
self.log.info("snapshot.context.ftable[test-event]=%s",
snapshot.context.ftable["test-event"])
self.log.info("snapshot.name= %s", snapshot.name)
self.log.info("snapshot.epoch= %s", snapshot.epoch)
self.log.info("==================================")
def take_snapshot(self, container):
"""
To take a snapshot on the container on current epoch.
Args:
container: container for the snapshot
Return:
An object representing the snapshot
"""
self.log.info("==Taking snapshot for:")
self.log.info(" coh= %s", container.coh)
snapshot = DaosSnapshot(self.context)
snapshot.create(container.coh)
self.display_snapshot(snapshot)
return snapshot
def invalid_snapshot_test(self, coh):
"""
Negative snapshot test with invalid container handle.
Args:
container: container for the snapshot
Return:
0: Failed
1: Passed (expected failure detected)
"""
status = 0
try:
snapshot = DaosSnapshot(self.context)
snapshot.create(coh)
except Exception as error:
self.log.info("==>Negative test, expected error: %s", str(error))
status = 1
return status
def test_snapshot_negativecases(self):
# pylint: disable=no-member
"""
Test ID: DAOS-1390 Verify snap_create bad parameter behavior.
DAOS-1322 Create a new container, verify snapshot state.
as expected for a brand new container.
DAOS-1392 Verify snap_destroy bad parameter behavior.
DAOS-1388 Verify snap_list bad parameter behavior.
Test Description:
(0)Take a snapshot of the newly created container.
(1)Create an object, write random data into it, and take
a snapshot.
(2)Verify the snapshot is working properly.
(3)Test snapshot with an invalid container handle.
(4)Test snapshot with a NULL container handle.
(5)Verify snap_destroy with a bad parameter.
(6)Verify snap_list bad parameter behavior.
Use Cases: Combinations with minimum 1 client and 1 server.
:avocado: tags=all,small,smoke,daily_regression,snap,snapshot_negative,
:avocado: tags=snapshotcreate_negative
"""
#DAOS-1322 Create a new container, verify snapshot state as expected
# for a brand new container.
try:
self.log.info(
"==(0)Take a snapshot of the newly created container.")
snapshot = DaosSnapshot(self.context)
snapshot.create(self.container.coh)
self.display_snapshot(snapshot)
except Exception as error:
self.fail("##(0)Error on a snapshot on a new container %s",
str(error))
#(1)Create an object, write some data into it, and take a snapshot
obj_cls = self.params.get("obj_class", '/run/object_class/*')
akey = self.params.get("akey", '/run/snapshot/*', default="akey")
dkey = self.params.get("dkey", '/run/snapshot/*', default="dkey")
data_size = self.params.get("test_datasize",
'/run/snapshot/*',
default=150)
rand_str = lambda n: ''.join(
[random.choice(string.lowercase) for i in range(n)])
thedata = "--->>>Happy Daos Snapshot-Create Negative Testing " + \
"<<<---" + rand_str(random.randint(1, data_size))
try:
obj = self.container.write_an_obj(thedata,
len(thedata) + 1,
dkey,
akey,
obj_cls=obj_cls)
except DaosApiError as error:
self.fail("##(1)Test failed during the initial object write: %s",
str(error))
obj.close()
##Take a snapshot of the container
snapshot = self.take_snapshot(self.container)
self.log.info("==(1)snapshot.epoch= %s", snapshot.epoch)
#(2)Verify the snapshot is working properly.
try:
obj.open()
snap_handle = snapshot.open(self.container.coh, snapshot.epoch)
thedata2 = self.container.read_an_obj(len(thedata) + 1,
dkey,
akey,
obj,
txn=snap_handle.value)
except Exception as error:
self.fail("##(2)Error when retrieving the snapshot data: %s",
str(error))
self.log.info("==(2)snapshot_list[ind]=%s", snapshot)
self.log.info("==snapshot.epoch= %s", snapshot.epoch)
self.log.info("==written thedata=%s", thedata)
self.log.info("==thedata2.value= %s", thedata2.value)
if thedata2.value != thedata:
raise Exception("##(2)The data in the snapshot is not the "
"same as the original data")
self.log.info("==Snapshot data matches the data originally "
"written.")
#(3)Test snapshot with an invalid container handle
self.log.info("==(3)Snapshot with an invalid container handle.")
if self.invalid_snapshot_test(self.container):
self.log.info(
"==>Negative test 1, expecting failed on taking "
"snapshot with an invalid container.coh: %s", self.container)
else:
self.fail(
"##(3)Negative test 1 passing, expecting failed on"
" taking snapshot with an invalid container.coh: %s",
self.container)
#(4)Test snapshot with a NULL container handle
self.log.info("==(4)Snapshot with a NULL container handle.")
if self.invalid_snapshot_test(None):
self.log.info("==>Negative test 2, expecting failed on taking "
"snapshot on a NULL container.coh.")
else:
self.fail("##(4)Negative test 2 passing, expecting failed on "
"taking snapshot with a NULL container.coh.")
#(5)DAOS-1392 destroy snapshot with an invalid handle
self.log.info(
"==(6)DAOS-1392 destroy snapshot with an invalid handle.")
try:
snapshot.destroy(None, snapshot.epoch)
self.fail("##(6)Negative test destroy snapshot with an "
"invalid coh handle, expected fail, shown Passing##")
except Exception as error:
self.log.info(
"==>Negative test, destroy snapshot with an invalid handle.")
self.log.info(" Expected Error: %s", str(error))
expected_error = "RC: -1002"
if expected_error not in str(error):
self.fail("##(6.1)Expecting error RC: -1002 did not show.")
#(6)DAOS-1388 Verify snap_list bad parameter behavior
self.log.info(
"==(7)DAOS-1388 Verify snap_list bad parameter behavior.")
try:
snapshot.list(None, 0)
self.fail("##(7)Negative test snapshot list with an "
"invalid coh and epoch, expected fail, shown Passing##")
except Exception as error:
self.log.info(
"==>Negative test, snapshot list with an invalid coh.")
self.log.info(" Expected Error: %s", str(error))
expected_error = "RC: -1002"
if expected_error not in str(error):
self.fail("##(7.1)Expecting error RC: -1002 did not show.")
def display_snapshot_test_data(self, test_data, ss_index):
"""Display the snapshot test data.
Args:
test_data: list of snapshot testdata
dictionary keys:
coh: container handle
snapshot: snapshot handle
tst_obj: test object
tst_data: test data
ss_index: snapshot-list index to be displayed.
"""
if len(test_data) < ss_index - 1:
self.log.info("##Under to display test_data info, "
"index out of range.")
else:
ind = ss_index - 1
self.log.info(" =Snapshot number : %s", ss_index)
self.log.info(" ==container_coh =%s", test_data[ind]["coh"])
self.log.info(" ==snapshot =%s",
test_data[ind]["snapshot"])
self.log.info(" ==snapshot.epoch =%s",
test_data[ind]["snapshot"].epoch)
self.log.info(" ==data obj =%s",
test_data[ind]["tst_obj"])
self.log.info(" ==snapshot tst_data_size= %s",
len(test_data[ind]["tst_data"]) + 1)
self.log.info(" ==original tst_data =%s",
test_data[ind]["tst_data"])
return
def test_snapshots(self):
# pylint: disable=no-member,too-many-locals
"""
Test ID: DAOS-1386 Test container SnapShot information
DAOS-1371 Test list snapshots
DAOS-1395 Test snapshot destroy
DAOS-1402 Test creating multiple snapshots
Test Description:
(1)Create an object, write random data into it, and take
a snapshot.
(2)Make changes to the data object. The write_an_obj function
does a commit when the update is complete.
(3)Verify the data in the snapshot is the original data.
Get a handle for the snapshot and read the object at dkey,
akey. Compare it to the originally written data.
(4)List the snapshot and make sure it reflects the original
epoch.
==>Repeat step(1) to step(4) for multiple snapshot tests.
(5)Verify the snapshots data.
(6)Destroy the snapshot individually.
(7)Check if still able to Open the destroyed snapshot and
Verify the snapshot removed from the snapshot list.
(8)Destroy the container snapshot.
Use Cases: Require 1 client and 1 server to run snapshot test.
1 pool and 1 container is used, num_of_snapshot defined
in the snapshot.yaml will be performed and verified.
:avocado: tags=all,small,smoke,snap,snapshots,full_regression
"""
test_data = []
ss_number = 0
obj_cls = self.params.get("obj_class", '/run/object_class/*')
akey = self.params.get("akey", '/run/snapshot/*', default="akey")
dkey = self.params.get("dkey", '/run/snapshot/*', default="dkey")
data_size = self.params.get("test_datasize",
'/run/snapshot/*',
default=150)
snapshot_loop = self.params.get("num_of_snapshot",
'/run/snapshot/*',
default=3)
rand_str = lambda n: ''.join(
[random.choice(string.lowercase) for i in range(n)])
#
#Test loop for creat, modify and snapshot object in the DAOS container.
#
while ss_number < snapshot_loop:
#(1)Create an object, write some data into it, and take a snapshot
ss_number += 1
thedata = "--->>>Happy Daos Snapshot Testing " + \
str(ss_number) + \
"<<<---" + rand_str(random.randint(1, data_size))
datasize = len(thedata) + 1
try:
obj = self.container.write_an_obj(thedata,
datasize,
dkey,
akey,
obj_cls=obj_cls)
obj.close()
except DaosApiError as error:
self.fail("##(1)Test failed during the initial object "
"write: {}".format(str(error)))
#Take a snapshot of the container
snapshot = DaosSnapshot(self.context)
snapshot.create(self.container.coh)
self.log.info("==Wrote an object and created a snapshot")
#Display snapshot
self.log.info("=(1.%s)snapshot test loop: %s", ss_number,
ss_number)
self.log.info(" ==snapshot.epoch= %s", snapshot.epoch)
self.display_snapshot(snapshot)
#Save snapshot test data
test_data.append({
"coh": self.container.coh,
"tst_obj": obj,
"snapshot": snapshot,
"tst_data": thedata
})
#(2)Make changes to the data object. The write_an_obj function does
# a commit when the update is complete
num_transactions = more_transactions = 200
self.log.info(
"=(2.%s)Committing %d additional transactions to "
"the same KV.", ss_number, more_transactions)
while more_transactions:
size = random.randint(1, 250) + 1
new_data = rand_str(size)
try:
new_obj = self.container.write_an_obj(new_data,
size,
dkey,
akey,
obj_cls=obj_cls)
new_obj.close()
except Exception as error:
self.fail("##(2)Test failed during the write of "
"multi-objects: {}".format(str(error)))
more_transactions -= 1
#(3)Verify the data in the snapshot is the original data.
# Get a handle for the snapshot and read the object at dkey, akey
# Compare it to the originally written data.
self.log.info("=(3.%s)snapshot test loop: %s", ss_number,
ss_number)
try:
obj.open()
snap_handle = snapshot.open(self.container.coh, snapshot.epoch)
thedata3 = self.container.read_an_obj(datasize,
dkey,
akey,
obj,
txn=snap_handle.value)
obj.close()
except Exception as error:
self.fail("##(3.1)Error when retrieving the snapshot data: {}".
format(str(error)))
self.display_snapshot_test_data(test_data, ss_number)
self.log.info(" ==thedata3.value= %s", thedata3.value)
if thedata3.value != thedata:
raise Exception("##(3.2)The data in the snapshot is not the "
"same as the original data")
self.log.info(" ==The snapshot data matches the data originally"
" written.")
#(4)List the snapshot and make sure it reflects the original epoch
try:
ss_list = snapshot.list(self.container.coh, snapshot.epoch)
self.log.info("=(4.%s)snapshot.list(self.container.coh)= %s",
ss_number, ss_list)
self.log.info(" ==snapshot.epoch= %s", snapshot.epoch)
except Exception as error:
self.fail(
"##(4)Test was unable to list the snapshot: {}".format(
str(error)))
self.log.info(
" ==After %s additional commits the snapshot is "
"still available", num_transactions)
#(5)Verify the snapshots data
for ind, _ in enumerate(test_data):
ss_number = ind + 1
self.log.info("=(5.%s)Verify the snapshot number %s:", ss_number,
ss_number)
self.display_snapshot_test_data(test_data, ss_number)
coh = test_data[ind]["coh"]
current_ss = test_data[ind]["snapshot"]
obj = test_data[ind]["tst_obj"]
tst_data = test_data[ind]["tst_data"]
datasize = len(tst_data) + 1
try:
obj.open()
snap_handle5 = snapshot.open(coh, current_ss.epoch)
thedata5 = self.container.read_an_obj(datasize,
dkey,
akey,
obj,
txn=snap_handle5.value)
obj.close()
except Exception as error:
self.fail("##(5.1)Error when retrieving the snapshot data: {}".
format(str(error)))
self.log.info(" ==snapshot tst_data =%s", thedata5.value)
if thedata5.value != tst_data:
raise Exception(
"##(5.2)Snapshot #%s, test data Mis-matches"
"the original data written.", ss_number)
self.log.info(
" snapshot test number %s, test data matches"
" the original data written.", ss_number)
#(6)Destroy the individual snapshot
self.log.info("=(6.%s)Destroy the snapshot epoch: %s", ss_number,
snapshot.epoch)
try:
snapshot.destroy(coh, snapshot.epoch)
self.log.info(" ==snapshot.epoch %s successfully destroyed",
snapshot.epoch)
except Exception as error:
self.fail("##(6)Error on snapshot.destroy: {}".format(
str(error)))
#(7)Check if still able to Open the destroyed snapshot and
# Verify the snapshot removed from the snapshot list
try:
obj.open()
snap_handle7 = snapshot.open(coh, snapshot.epoch)
thedata7 = self.container.read_an_obj(datasize,
dkey,
akey,
obj,
txn=snap_handle7.value)
obj.close()
except Exception as error:
self.fail(
"##(7)Error when retrieving the snapshot data: {}".format(
str(error)))
self.log.info("=(7)=>thedata_after_snapshot.destroyed.value= %s",
thedata7.value)
self.log.info(" ==>snapshot.epoch= %s", snapshot.epoch)
#Still able to open the snapshot and read data after destroyed.
try:
ss_list = snapshot.list(coh, snapshot.epoch)
self.log.info(" -->snapshot.list(coh, snapshot.epoch)= %s",
ss_list)
except Exception as error:
self.fail("##(7)Error when calling the snapshot list: {}".format(
str(error)))
#(8)Destroy the snapshot on the container
try:
snapshot.destroy(coh)
self.log.info("=(8)Container snapshot destroyed successfully.")
except Exception as error:
self.fail("##(8)Error on snapshot.destroy. {}".format(str(error)))
self.log.info("===DAOS container Multiple snapshots test passed.")
class FullPoolContainerCreate(TestWithServers):
"""
Class for test to create a container in a pool with no remaining free space.
:avocado: recursive
"""
def __init__(self, *args, **kwargs):
super(FullPoolContainerCreate, self).__init__(*args, **kwargs)
self.cont = None
self.cont2 = None
@skipForTicket("DAOS-3142")
def test_no_space_cont_create(self):
"""
:avocado: tags=all,container,tiny,full_regression,fullpoolcontcreate
"""
# full storage rc
err = "-1007"
# probably should be -1007, revisit later
err2 = "-1009"
# create pool
self.pool = DaosPool(self.context)
mode = self.params.get("mode", '/conttests/createmode/')
self.d_log.debug("mode is {0}".format(mode))
uid = os.geteuid()
gid = os.getegid()
# 16 mb pool, minimum size currently possible
size = 16777216
self.d_log.debug("creating pool")
self.pool.create(mode, uid, gid, size, self.server_group, None)
self.d_log.debug("created pool")
# connect to the pool
self.d_log.debug("connecting to pool")
self.pool.connect(1 << 1)
self.d_log.debug("connected to pool")
# query the pool
self.d_log.debug("querying pool info")
dummy_pool_info = self.pool.pool_query()
self.d_log.debug("queried pool info")
# create a container
try:
self.d_log.debug("creating container")
self.cont = DaosContainer(self.context)
self.cont.create(self.pool.handle)
self.d_log.debug("created container")
except DaosApiError as excep:
self.d_log.error("caught exception creating container: "
"{0}".format(excep))
self.fail("caught exception creating container: {0}".format(excep))
self.d_log.debug("opening container")
self.cont.open()
self.d_log.debug("opened container")
# generate random dkey, akey each time
# write 1mb until no space, then 1kb, etc. to fill pool quickly
for obj_sz in [1048576, 10240, 10, 1]:
write_count = 0
while True:
self.d_log.debug("writing obj {0}, sz {1} to "
"container".format(write_count, obj_sz))
my_str = "a" * obj_sz
my_str_sz = obj_sz
dkey = (''.join(
random.choice(string.lowercase) for i in range(5)))
akey = (''.join(
random.choice(string.lowercase) for i in range(5)))
try:
dummy_oid = self.cont.write_an_obj(my_str,
my_str_sz,
dkey,
akey,
obj_cls="OC_SX")
self.d_log.debug("wrote obj {0}, sz {1}".format(
write_count, obj_sz))
write_count += 1
except DaosApiError as excep:
if not (err in repr(excep) or err2 in repr(excep)):
self.d_log.error("caught exception while writing "
"object: {0}".format(repr(excep)))
self.fail("caught exception while writing object: {0}".
format(repr(excep)))
else:
self.d_log.debug("pool is too full for {0} byte "
"objects".format(obj_sz))
break
self.d_log.debug("closing container")
self.cont.close()
self.d_log.debug("closed container")
# create a 2nd container now that pool is full
try:
self.d_log.debug("creating 2nd container")
self.cont2 = DaosContainer(self.context)
self.cont2.create(self.pool.handle)
self.d_log.debug("created 2nd container")
self.d_log.debug("opening container 2")
self.cont2.open()
self.d_log.debug("opened container 2")
self.d_log.debug("writing one more object, write expected to fail")
self.cont2.write_an_obj(my_str,
my_str_sz,
dkey,
akey,
obj_cls="OC_SX")
self.fail("wrote one more object after pool was completely filled,"
" this should never print")
except DaosApiError as excep:
if not (err in repr(excep) or err2 in repr(excep)):
self.d_log.error("caught unexpected exception while "
"writing object: {0}".format(repr(excep)))
self.fail("caught unexpected exception while writing "
"object: {0}".format(repr(excep)))
else:
self.d_log.debug("correctly caught -1007 while attempting "
"to write object in full pool")
class OSAUtils(IorTestBase):
# pylint: disable=too-many-ancestors
"""
Test Class Description: This test runs
daos_server offline drain test cases.
:avocado: recursive
"""
def setUp(self):
"""Set up for test case."""
super(OSAUtils, self).setUp()
self.container = None
self.obj = None
self.ioreq = None
self.dmg_command = self.get_dmg_command()
self.no_of_dkeys = self.params.get("no_of_dkeys",
'/run/dkeys/*',
default=[0])[0]
self.no_of_akeys = self.params.get("no_of_akeys",
'/run/akeys/*',
default=[0])[0]
self.record_length = self.params.get("length",
'/run/record/*',
default=[0])[0]
@fail_on(CommandFailure)
def get_pool_leader(self):
"""Get the pool leader.
Returns:
int: pool leader value
"""
data = self.dmg_command.pool_query(self.pool.uuid)
return int(data["leader"])
@fail_on(CommandFailure)
def get_rebuild_status(self):
"""Get the rebuild status.
Returns:
str: reuild status
"""
data = self.dmg_command.pool_query(self.pool.uuid)
return data["rebuild"]["status"]
@fail_on(CommandFailure)
def is_rebuild_done(self, time_interval):
"""Rebuild is completed/done.
Args:
time_interval: Wait interval between checks
Returns:
False: If rebuild_status not "done" or "completed".
True: If rebuild status is "done" or "completed".
"""
status = False
fail_count = 0
completion_flag = ["done", "completed"]
while fail_count <= 20:
rebuild_status = self.get_rebuild_status()
time.sleep(time_interval)
fail_count += 1
if rebuild_status in completion_flag:
status = True
break
return status
@fail_on(CommandFailure)
def assert_on_rebuild_failure(self):
"""If the rebuild is not successful,
raise assert.
"""
rebuild_status = self.get_rebuild_status()
self.log.info("Rebuild Status: %s", rebuild_status)
rebuild_failed_string = ["failed", "scanning", "aborted", "busy"]
self.assertTrue(rebuild_status not in rebuild_failed_string,
"Rebuild failed")
@fail_on(CommandFailure)
def get_pool_version(self):
"""Get the pool version.
Returns:
int: pool_version_value
"""
data = self.dmg_command.pool_query(self.pool.uuid)
return int(data["version"])
@fail_on(DaosApiError)
def write_single_object(self):
"""Write some data to the existing pool."""
self.pool.connect(2)
csum = self.params.get("enable_checksum", '/run/container/*')
self.container = DaosContainer(self.context)
input_param = self.container.cont_input_values
input_param.enable_chksum = csum
self.container.create(poh=self.pool.pool.handle, con_prop=input_param)
self.container.open()
self.obj = DaosObj(self.context, self.container)
self.obj.create(objcls=1)
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj,
objtype=4)
self.log.info("Writing the Single Dataset")
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0]) * self.record_length)
d_key_value = "dkey {0}".format(dkey)
c_dkey = ctypes.create_string_buffer(d_key_value)
a_key_value = "akey {0}".format(akey)
c_akey = ctypes.create_string_buffer(a_key_value)
c_value = ctypes.create_string_buffer(indata)
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size)
self.obj.close()
self.container.close()
@fail_on(DaosApiError)
def verify_single_object(self):
"""Verify the container data on the existing pool."""
self.pool.connect(2)
self.container.open()
self.obj.open()
self.log.info("Single Dataset Verification -- Started")
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0]) * self.record_length)
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1)
if indata != (repr(val.value)[1:-1]):
self.d_log.error("ERROR:Data mismatch for "
"dkey = {0}, "
"akey = {1}".format(
"dkey {0}".format(dkey),
"akey {0}".format(akey)))
self.fail(
"ERROR: Data mismatch for dkey = {0}, akey={1}".format(
"dkey {0}".format(dkey), "akey {0}".format(akey)))
self.obj.close()
self.container.close()
def ior_thread(self, pool, oclass, api, test, flags, results):
"""Start threads and wait until all threads are finished.
Args:
pool (object): pool handle
oclass (str): IOR object class
api (str): IOR api
test (list): IOR test sequence
flags (str): IOR flags
results (queue): queue for returning thread results
"""
container_info = {}
mpio_util = MpioUtils()
if mpio_util.mpich_installed(self.hostlist_clients) is False:
self.fail("Exiting Test : Mpich not installed on :"
" {}".format(self.hostfile_clients[0]))
self.pool = pool
# Define the arguments for the ior_runner_thread method
ior_cmd = IorCommand()
ior_cmd.get_params(self)
ior_cmd.set_daos_params(self.server_group, self.pool)
ior_cmd.dfs_oclass.update(oclass)
ior_cmd.api.update(api)
ior_cmd.transfer_size.update(test[2])
ior_cmd.block_size.update(test[3])
ior_cmd.flags.update(flags)
container_info["{}{}{}".format(oclass, api,
test[2])] = str(uuid.uuid4())
# Define the job manager for the IOR command
self.job_manager = Mpirun(ior_cmd, mpitype="mpich")
key = "".join([oclass, api, str(test[2])])
self.job_manager.job.dfs_cont.update(container_info[key])
env = ior_cmd.get_default_env(str(self.job_manager))
self.job_manager.assign_hosts(self.hostlist_clients, self.workdir,
None)
self.job_manager.assign_processes(self.processes)
self.job_manager.assign_environment(env, True)
# run IOR Command
try:
self.job_manager.run()
except CommandFailure as _error:
results.put("FAIL")
class BasicSnapshot(TestWithServers):
"""DAOS-1370 Basic snapshot test.
Test Class Description:
Test that a snapshot taken of a container remains unchaged even after
an object in the container has been updated 500 times.
Create the container.
Write an object to the container.
Take a snapshot.
Write 500 changes to the KV pair of the object.
Check that the snapshot is still there.
Confirm that the data in the snapshot is unchanged.
Destroy the snapshot
:avocado: recursive
"""
def __init__(self, *args, **kwargs):
"""Initialize a BasicSnapshot object."""
super(BasicSnapshot, self).__init__(*args, **kwargs)
self.snapshot = None
def test_basic_snapshot(self):
"""Test ID: DAOS-1370.
Test Description:
Create a pool, container in the pool, object in the container, add
one key:value to the object.
Commit the transaction. Perform a snapshot create on the container.
Create 500 additional transactions with a small change to the object
in each and commit each after the object update is done.
Verify the snapshot is still available and the contents remain in
their original state.
:avocado: tags=snap,basicsnap
"""
# Set up the pool and container.
try:
# initialize a pool object then create the underlying
# daos storage, and connect
self.prepare_pool()
# create a container
self.container = DaosContainer(self.context)
self.container.create(self.pool.pool.handle)
# now open it
self.container.open()
except DaosApiError as error:
self.log.error(str(error))
self.fail("Test failed before snapshot taken")
try:
# create an object and write some data into it
obj_cls = self.params.get("obj_class", '/run/object_class/*')
thedata = "Now is the winter of our discontent made glorious"
datasize = len(thedata) + 1
dkey = "dkey"
akey = "akey"
tx_handle = self.container.get_new_tx()
obj = self.container.write_an_obj(thedata,
datasize,
dkey,
akey,
obj_cls=obj_cls,
txn=tx_handle)
self.container.commit_tx(tx_handle)
obj.close()
# Take a snapshot of the container
self.snapshot = DaosSnapshot(self.context)
self.snapshot.create(self.container.coh, tx_handle)
self.log.info("Wrote an object and created a snapshot")
except DaosApiError as error:
self.fail("Test failed during the initial object write.\n{0}"
.format(error))
# Make 500 changes to the data object. The write_an_obj function does a
# commit when the update is complete
try:
self.log.info(
"Committing 500 additional transactions to the same KV")
more_transactions = 500
while more_transactions:
size = random.randint(1, 250) + 1
new_data = get_random_string(size)
new_obj = self.container.write_an_obj(
new_data, size, dkey, akey, obj_cls=obj_cls)
new_obj.close()
more_transactions -= 1
except DaosApiError as error:
self.fail(
"Test failed during the write of 500 objects.\n{0}".format(
error))
# List the snapshot
try:
reported_epoch = self.snapshot.list(self.container.coh)
except DaosApiError as error:
self.fail(
"Test was unable to list the snapshot\n{0}".format(error))
# Make sure the snapshot reflects the original epoch
if self.snapshot.epoch != reported_epoch:
self.fail(
"The snapshot epoch returned from snapshot list is not the "
"same as the original epoch snapshotted.")
self.log.info(
"After 500 additional commits the snapshot is still available")
# Make sure the data in the snapshot is the original data.
# Get a handle for the snapshot and read the object at dkey, akey.
try:
obj.open()
snap_handle = self.snapshot.open(self.container.coh)
thedata2 = self.container.read_an_obj(
datasize, dkey, akey, obj, txn=snap_handle.value)
except DaosApiError as error:
self.fail(
"Error when retrieving the snapshot data.\n{0}".format(error))
# Compare the snapshot to the originally written data.
if thedata2.value != thedata:
self.fail(
"The data in the snapshot is not the same as the original data")
self.log.info(
"The snapshot data matches the data originally written.")
# Now destroy the snapshot
try:
self.snapshot.destroy(self.container.coh)
self.log.info("Snapshot successfully destroyed")
except DaosApiError as error:
self.fail(str(error))
class Permission(TestWithServers):
"""Test pool permissions.
Test Class Description:
Tests DAOS pool permissions while connect, whether
modifying file with specific permissions work as expected.
:avocado: recursive
"""
# Cancel any tests with tickets already assigned
CANCEL_FOR_TICKET = [
["DAOS-3442", "mode", 73],
["DAOS-3442", "mode", 146, "perm", 0],
["DAOS-3442", "mode", 146, "perm", 2],
["DAOS-3442", "mode", 292],
]
def test_connectpermission(self):
"""Test ID: DAOS-???.
Test Description:
Test pool connections with specific permissions.
:avocado: tags=pool,permission,connectpermission
"""
# parameter used in pool create
createmode = self.params.get("mode", '/run/createtests/createmode/*/')
# parameter used for pool connect
permissions = self.params.get("perm", '/run/createtests/permissions/*')
if createmode == 73:
expected_result = RESULT_FAIL
if createmode == 511 and permissions == 0:
expected_result = RESULT_PASS
elif createmode in [146, 511] and permissions == 1:
expected_result = RESULT_PASS
elif createmode in [292, 511] and permissions == 2:
expected_result = RESULT_PASS
else:
expected_result = RESULT_FAIL
# initialize a python pool object then create the underlying
# daos storage
self.pool = TestPool(self.context, self.get_dmg_command())
self.test_log.debug("Pool initialization successful")
self.pool.get_params(self)
self.pool.mode.value = createmode
self.pool.create()
self.test_log.debug("Pool Creation successful")
try:
self.pool.connect(1 << permissions)
self.test_log.debug("Pool Connect successful")
if expected_result == RESULT_FAIL:
self.fail(
"Test was expected to fail at pool.connect, but it " +
"passed.\n")
except TestFail as excep:
self.log.error(str(excep))
if expected_result == RESULT_PASS:
self.fail("Test was expected to pass but it failed at " +
"pool.connect.\n")
def test_filemodification(self):
"""Test ID: DAOS-???.
Test Description:
Test whether file modification happens as expected under different
permission levels.
:avocado: tags=pool,permission,filemodification
"""
# parameters used in pool create
createmode = self.params.get("mode", '/run/createtests/createmode/*/')
if createmode == 73:
expected_result = RESULT_FAIL
elif createmode in [146, 511]:
permissions = 1
expected_result = RESULT_PASS
elif createmode == 292:
permissions = 2
expected_result = RESULT_PASS
# initialize a python pool object then create the underlying
# daos storage
self.pool = TestPool(self.context, self.get_dmg_command())
self.test_log.debug("Pool initialization successful")
self.pool.get_params(self)
self.pool.mode.value = createmode
self.test_log.debug("Pool Creation successful")
try:
self.pool.connect(1 << permissions)
self.test_log.debug("Pool Connect successful")
if expected_result == RESULT_FAIL:
self.fail("Test was expected to fail at pool.connect but it " +
"passed.\n")
except TestFail as excep:
self.log.error(str(excep))
if expected_result == RESULT_PASS:
self.fail("Test was expected to pass but it failed at " +
"pool.connect.\n")
try:
self.container = DaosContainer(self.context)
self.test_log.debug("Container initialization successful")
self.container.create(self.pool.pool.handle)
self.test_log.debug("Container create successful")
# now open it
self.container.open()
self.test_log.debug("Container open successful")
thedata = "a string that I want to stuff into an object"
size = 45
dkey = "this is the dkey"
akey = "this is the akey"
self.container.write_an_obj(thedata, size, dkey, akey)
self.test_log.debug("Container write successful")
if expected_result == RESULT_FAIL:
self.fail(
"Test was expected to fail at container operations " +
"but it passed.\n")
except DaosApiError as excep:
self.log.error(str(excep))
if expected_result == RESULT_PASS:
self.fail(
"Test was expected to pass but it failed at container " +
"operations.\n")
class ObjFetchBadParam(TestWithServers):
"""
Test Class Description:
Pass an assortment of bad parameters to the daos_obj_fetch function.
:avocado: recursive
"""
def setUp(self):
super(ObjFetchBadParam, self).setUp()
time.sleep(5)
self.prepare_pool()
try:
# create a container
self.container = DaosContainer(self.context)
self.container.create(self.pool.pool.handle)
# now open it
self.container.open()
# create an object and write some data into it
thedata = "a string that I want to stuff into an object"
self.datasize = len(thedata) + 1
self.dkey = "this is the dkey"
self.akey = "this is the akey"
self.obj = self.container.write_an_obj(thedata, self.datasize,
self.dkey, self.akey, None,
None, 2)
thedata2 = self.container.read_an_obj(self.datasize, self.dkey,
self.akey, self.obj)
if thedata not in thedata2.value:
print(thedata)
print(thedata2.value)
self.fail("Error reading back data, test failed during"\
" the initial setup.\n")
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
self.fail("Test failed during the initial setup.\n")
def test_bad_handle(self):
"""
Test ID: DAOS-1377
Test Description: Pass a bogus object handle, should return bad handle.
:avocado: tags=all,object,full_regression,small,objbadhandle
"""
try:
# trash the handle and read again
saved_oh = self.obj.obj_handle
self.obj.obj_handle = 99999
# expecting this to fail with -1002
dummy_thedata2 = self.container.read_an_obj(
self.datasize, self.dkey, self.akey, self.obj)
self.container.oh = saved_oh
self.fail("Test was expected to return a -1002 but it has not.\n")
except DaosApiError as excep:
self.container.oh = saved_oh
if '-1002' not in str(excep):
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1002 but it has not.\n")
def test_null_ptrs(self):
"""
Test ID: DAOS-1377
Test Description: Pass null pointers for various fetch parameters.
:avocado: tags=all,object,full_regression,small,objfetchnull
"""
try:
# now try it with a bad dkey, expecting this to fail with -1003
dummy_thedata2 = self.container.read_an_obj(
self.datasize, None, self.akey, self.obj)
self.container.close()
self.container.destroy()
self.pool.destroy(1)
self.fail("Test was expected to return a -1003 but it has not.\n")
except DaosApiError as excep:
if '-1003' not in str(excep):
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1003 but it has not.\n")
try:
# now try it with a null sgl (iod_size is not set)
# expecting this to fail with -2013
test_hints = ['sglnull']
dummy_thedata2 = self.container.read_an_obj(
self.datasize, self.dkey, self.akey, self.obj, test_hints)
# behavior not as expect so commented out for now
# when DAOS-1448 is complete, uncomment and retest
self.fail("Test was expected to return a -2013 but it has not.\n")
except DaosApiError as excep:
if '-2013' not in str(excep):
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -2013 but it has not.\n")
try:
# when DAOS-1449 is complete, uncomment and retest
# now try it with a null iod, expecting this to fail with -1003
#test_hints = ['iodnull']
#thedata2 = self.container.read_an_obj(self.datasize, dkey, akey,
# self.obj, test_hints)
pass
#self.fail("Test was expected to return a -1003 but it has not.\n")
except DaosApiError as excep:
if '-1003' not in str(excep):
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1003 but it has not.\n")
def test_array_obj(self):
"""
Test ID: DAOS-961
Test Description: Writes an array to an object and then reads it
back and verifies it.
:avocado: tags=all,smoke,pr,object,tiny,basicobject
"""
self.prepare_pool()
try:
# create a container
container = DaosContainer(self.context)
container.create(self.pool.pool.handle)
self.plog.info("Container %s created.", container.get_uuid_str())
# now open it
container.open()
# do a query and compare the UUID returned from create with
# that returned by query
container.query()
if container.get_uuid_str() != c_uuid_to_str(
container.info.ci_uuid):
self.fail("Container UUID did not match the one in info\n")
# create an object and write some data into it
thedata = []
thedata.append("data string one")
thedata.append("data string two")
thedata.append("data string tre")
dkey = "this is the dkey"
akey = "this is the akey"
self.plog.info("writing array to dkey >%s< akey >%s<.", dkey, akey)
oid = container.write_an_array_value(thedata,
dkey,
akey,
obj_cls=3)
# read the data back and make sure its correct
length = len(thedata[0])
thedata2 = container.read_an_array(len(thedata), length + 1, dkey,
akey, oid)
if thedata[0][0:length - 1] != thedata2[0][0:length - 1]:
self.plog.error("Data mismatch")
self.plog.error("Wrote: >%s<", thedata[0])
self.plog.error("Read: >%s<", thedata2[0])
self.fail("Write data, read it back, didn't match\n")
if thedata[2][0:length - 1] != thedata2[2][0:length - 1]:
self.plog.error("Data mismatch")
self.plog.error("Wrote: >%s<", thedata[2])
self.plog.error("Read: >%s<", thedata2[2])
self.fail("Write data, read it back, didn't match\n")
container.close()
# wait a few seconds and then destroy
time.sleep(5)
container.destroy()
self.plog.info("Test Complete")
except DaosApiError as excep:
self.plog.error("Test Failed, exception was thrown.")
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to pass but it failed.\n")
class DeleteContainerTest(TestWithServers):
"""
Tests DAOS container delete and close.
:avocado: recursive
"""
def setUp(self):
super(DeleteContainerTest, self).setUp()
# parameters used in pool create
self.createmode = self.params.get("mode",
'/run/createtests/createmode/')
self.createuid = os.geteuid()
self.creategid = os.getegid()
self.createsetid = self.params.get("setname",
'/run/createtests/createset/')
self.createsize = self.params.get("size",
'/run/createtests/createsize/')
def test_container_delete(self):
"""
Test basic container delete
:avocado: tags=all,container,tiny,smoke,pr,contdelete
"""
expected_for_param = []
uuidlist = self.params.get("uuid",
'/run/createtests/ContainerUUIDS/*/')
cont_uuid = uuidlist[0]
expected_for_param.append(uuidlist[1])
pohlist = self.params.get("poh", '/run/createtests/PoolHandles/*/')
poh = pohlist[0]
expected_for_param.append(pohlist[1])
openlist = self.params.get("opened",
"/run/createtests/ConnectionOpened/*/")
opened = openlist[0]
expected_for_param.append(openlist[1])
forcelist = self.params.get("force",
"/run/createtests/ForceDestroy/*/")
force = forcelist[0]
expected_for_param.append(forcelist[1])
if force >= 1:
self.cancel("Force >= 1 blocked by issue described in "
"https://jira.hpdd.intel.com/browse/DAOS-689")
if force == 0:
self.cancel("Force = 0 blocked by "
"https://jira.hpdd.intel.com/browse/DAOS-1935")
expected_result = 'PASS'
for result in expected_for_param:
if result == 'FAIL':
expected_result = 'FAIL'
break
try:
# initialize a python pool object then create the underlying
# daos storage
self.pool = DaosPool(self.context)
self.pool.create(self.createmode, self.createuid, self.creategid,
self.createsize, self.createsetid, None)
# need a connection to create container
self.pool.connect(1 << 1)
self.container = DaosContainer(self.context)
# create should always work (testing destroy)
if not cont_uuid == 'INVALID':
cont_uuid = uuid.UUID(uuidlist[0])
self.container.create(self.pool.handle, cont_uuid)
else:
self.container.create(self.pool.handle)
# Opens the container if required
if opened:
self.container.open(self.pool.handle)
# wait a few seconds and then attempts to destroy container
time.sleep(5)
if poh == 'VALID':
poh = self.pool.handle
# if container is INVALID, overwrite with non existing UUID
if cont_uuid == 'INVALID':
cont_uuid = uuid.uuid4()
self.container.destroy(force=force, poh=poh, con_uuid=cont_uuid)
self.container = None
if expected_result in ['FAIL']:
self.fail("Test was expected to fail but it passed.\n")
except DaosApiError as excep:
self.d_log.error(excep)
self.d_log.error(traceback.format_exc())
if expected_result == 'PASS':
self.fail("Test was expected to pass but it failed.\n")
class ObjectDataValidation(avocado.Test):
"""
Test Class Description:
Tests that create Different length records,
Disconnect the pool/container and reconnect,
validate the data after reconnect.
"""
# pylint: disable=too-many-instance-attributes
def setUp(self):
self.agent_sessions = None
self.pool = None
self.container = None
self.obj = None
self.ioreq = None
self.hostlist = None
self.hostfile = None
self.no_of_dkeys = None
self.no_of_akeys = None
self.array_size = None
self.record_length = None
with open('../../.build_vars.json') as json_f:
build_paths = json.load(json_f)
self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../")
server_group = self.params.get("name", '/server_config/',
'daos_server')
self.context = DaosContext(build_paths['PREFIX'] + '/lib64/')
self.d_log = DaosLog(self.context)
self.hostlist = self.params.get("test_machines", '/run/hosts/*')
self.hostfile = write_host_file.write_host_file(
self.hostlist, self.workdir)
self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0]
self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0]
self.array_size = self.params.get("size", '/array_size/')
self.record_length = self.params.get("length", '/run/record/*')
self.agent_sessions = agent_utils.run_agent(self.basepath,
self.hostlist)
server_utils.run_server(self, self.hostfile, server_group)
self.pool = DaosPool(self.context)
self.pool.create(self.params.get("mode", '/run/pool/createmode/*'),
os.geteuid(), os.getegid(),
self.params.get("size", '/run/pool/createsize/*'),
self.params.get("setname", '/run/pool/createset/*'),
None)
self.pool.connect(2)
self.container = DaosContainer(self.context)
self.container.create(self.pool.handle)
self.container.open()
self.obj = DaosObj(self.context, self.container)
self.obj.create(objcls=1)
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj,
objtype=4)
def tearDown(self):
try:
if self.container:
self.container.close()
self.container.destroy()
if self.pool:
self.pool.disconnect()
self.pool.destroy(1)
finally:
if self.agent_sessions:
agent_utils.stop_agent(self.agent_sessions)
server_utils.stop_server(hosts=self.hostlist)
def reconnect(self):
'''
Function to reconnect the pool/container and reopen the Object
for read verification.
'''
#Close the Obj/Container, Disconnect the Pool.
self.obj.close()
self.container.close()
self.pool.disconnect()
time.sleep(5)
#Connect Pool, Open Container and Object
self.pool.connect(2)
self.container.open()
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj,
objtype=4)
@avocado.fail_on(DaosApiError)
def test_invalid_tx_commit_close(self):
"""
Test ID:
(1)DAOS-1346: Verify commit tx bad parameter behavior.
(2)DAOS-1343: Verify tx_close bad parameter behavior.
(3)DAOS-1342: Verify tx_close through daos_api.
(4)DAOS-1338: Add and verify tx_abort through daos_api.
(5)DAOS-1339: Verify tx_abort bad parameter behavior.
Test Description:
Write Avocado Test to verify commit tx and close tx
bad parameter behavior.
:avocado: tags=all,object,full_regression,small,invalid_tx
"""
self.d_log.info("==Writing the Single Dataset for negative test...")
record_index = 0
expected_error = "RC: -1002"
dkey = 0
akey = 0
indata = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
c_value = ctypes.create_string_buffer(indata)
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
try:
new_transaction = self.container.get_new_tx()
except DaosApiError as excep:
#initial container get_new_tx failed, skip rest of the test
self.fail("##container get_new_tx failed: {}".format(excep))
invalid_transaction = new_transaction + random.randint(1000, 383838)
self.log.info("==new_transaction= %s", new_transaction)
self.log.info("==invalid_transaction= %s", invalid_transaction)
self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
new_transaction)
try:
self.container.commit_tx(invalid_transaction)
self.fail("##(1.1)Container.commit_tx passing with invalid handle")
except DaosApiError as excep:
self.log.info(str(excep))
self.log.info(
"==(1)Expecting failure: invalid Container.commit_tx.")
if expected_error not in str(excep):
self.fail(
"##(1.2)Expecting error RC: -1002, but got {}.".format(
str(excep)))
try:
self.container.close_tx(invalid_transaction)
self.fail("##(2.1)Container.close_tx passing with invalid handle")
except DaosApiError as excep:
self.log.info(str(excep))
self.log.info(
"==(2)Expecting failure: invalid Container.commit_tx.")
if expected_error not in str(excep):
self.fail(
"##(2.2)Expecting error RC: -1002, but got {}.".format(
str(excep)))
try:
self.container.close_tx(new_transaction)
self.log.info("==(3)container.close_tx test passed.")
except DaosApiError as excep:
self.log.info(str(excep))
self.fail("##(3)Failed on close_tx.")
try:
self.container.abort_tx(invalid_transaction)
self.fail("##(4.1)Container.abort_tx passing with invalid handle")
except DaosApiError as excep:
self.log.info(str(excep))
self.log.info(
"==(4)Expecting failure: invalid Container.abort_tx.")
if expected_error not in str(excep):
self.fail(
"##(4.2)Expecting error RC: -1002, but got {}.".format(
str(excep)))
#Try to abort the transaction which already closed.
try:
self.container.abort_tx(new_transaction)
self.fail("##(5.1)Container.abort_tx passing with a closed handle")
except DaosApiError as excep:
self.log.info(str(excep))
self.log.info(
"==(5)Expecting failure: Container.abort_tx closed handle.")
if expected_error not in str(excep):
self.fail(
"##(5.2)Expecting error RC: -1002, but got {}.".format(
str(excep)))
#open another transaction for abort test
try:
new_transaction2 = self.container.get_new_tx()
except DaosApiError as excep:
self.fail("##(6.1)container get_new_tx failed: {}".format(excep))
self.log.info("==new_transaction2= %s", new_transaction2)
self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
new_transaction2)
try:
self.container.abort_tx(new_transaction2)
self.log.info("==(6)container.abort_tx test passed.")
except DaosApiError as excep:
self.log.info(str(excep))
self.fail("##(6.2)Failed on abort_tx.")
@avocado.fail_on(DaosApiError)
@skipForTicket("DAOS-3208")
def test_single_object_validation(self):
"""
Test ID: DAOS-707
Test Description: Write Avocado Test to verify single data after
pool/container disconnect/reconnect.
:avocado: tags=all,object,full_regression,small,single_object
"""
self.d_log.info("Writing the Single Dataset")
record_index = 0
transaction = []
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
c_value = ctypes.create_string_buffer(indata)
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
new_transaction = self.container.get_new_tx()
self.ioreq.single_insert(c_dkey, c_akey, c_value, c_size,
new_transaction)
self.container.commit_tx(new_transaction)
transaction.append(new_transaction)
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
self.reconnect()
self.d_log.info("Single Dataset Verification -- Started")
record_index = 0
transaction_index = 0
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
val = self.ioreq.single_fetch(c_dkey, c_akey, len(indata) + 1)
if indata != (repr(val.value)[1:-1]):
self.d_log.error("ERROR:Data mismatch for "
"dkey = {0}, "
"akey = {1}".format(
"dkey {0}".format(dkey),
"akey {0}".format(akey)))
self.fail(
"ERROR: Data mismatch for dkey = {0}, akey={1}".format(
"dkey {0}".format(dkey), "akey {0}".format(akey)))
transaction_index = transaction_index + 1
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
@avocado.fail_on(DaosApiError)
@skipForTicket("DAOS-3208")
def test_array_object_validation(self):
"""
Test ID: DAOS-707
Test Description: Write Avocado Test to verify Array data after
pool/container disconnect/reconnect.
:avocado: tags=all,object,full_regression,small,array_object
"""
self.d_log.info("Writing the Array Dataset")
record_index = 0
transaction = []
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
c_values = []
value = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
for item in range(self.array_size):
c_values.append(
(ctypes.create_string_buffer(value), len(value) + 1))
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
new_transaction = self.container.get_new_tx()
self.ioreq.insert_array(c_dkey, c_akey, c_values,
new_transaction)
self.container.commit_tx(new_transaction)
transaction.append(new_transaction)
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
self.reconnect()
self.d_log.info("Array Dataset Verification -- Started")
record_index = 0
transaction_index = 0
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = []
value = ("{0}".format(str(akey)[0]) *
self.record_length[record_index])
for item in range(self.array_size):
indata.append(value)
c_dkey = ctypes.create_string_buffer("dkey {0}".format(dkey))
c_akey = ctypes.create_string_buffer("akey {0}".format(akey))
c_rec_count = ctypes.c_uint(len(indata))
c_rec_size = ctypes.c_size_t(len(indata[0]) + 1)
outdata = self.ioreq.fetch_array(c_dkey, c_akey, c_rec_count,
c_rec_size)
for item in enumerate(indata):
if indata[item[0]] != outdata[item[0]][:-1]:
self.d_log.error("ERROR:Data mismatch for "
"dkey = {0}, "
"akey = {1}".format(
"dkey {0}".format(dkey),
"akey {0}".format(akey)))
self.fail(
"ERROR:Data mismatch for dkey = {0}, akey={1}".
format("dkey {0}".format(dkey),
"akey {0}".format(akey)))
transaction_index = transaction_index + 1
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
class ObjUpdateBadParam(TestWithServers):
"""
Test Class Description:
Pass an assortment of bad parameters to the daos_obj_update function.
:avocado: recursive
"""
def setUp(self):
super().setUp()
self.plog = logging.getLogger("progress")
try:
self.prepare_pool()
# create a container
self.container = DaosContainer(self.context)
self.container.create(self.pool.pool.handle)
self.plog.info("Container %s created.",
self.container.get_uuid_str())
# now open it
self.container.open()
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
self.fail("Test failed during setup .\n")
def test_bad_handle(self):
"""
Test ID: DAOS-1376
Test Description: Pass a bogus object handle, should return bad handle.
:avocado: tags=all,full_regression,small
:avocado: tags=object,objupdatebadparam
:avocado: tags=objbadhand
"""
try:
# create an object and write some data into it
thedata = b"a string that I want to stuff into an object"
thedatasize = len(thedata) + 1
dkey = b"this is the dkey"
akey = b"this is the akey"
obj = self.container.write_an_obj(thedata, thedatasize, dkey, akey,
None, None, 2)
saved_oh = obj.obj_handle
obj.obj_handle = 99999
obj = self.container.write_an_obj(thedata, thedatasize, dkey, akey,
obj, None, 2)
self.container.oh = saved_oh
self.fail("Test was expected to return a -1002 but it has not.\n")
except DaosApiError as excep:
self.container.oh = saved_oh
self.plog.info("Test Complete")
if '-1002' not in str(excep):
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1002 but it has not.\n")
def test_null_values(self):
"""
Test ID: DAOS-1376
Test Description: Pass a dkey and an akey that is null.
:avocado: tags=all,full_regression,small
:avocado: tags=object,objupdatebadparam
:avocado: tags=objupdatenull
"""
# data used in the test
thedata = b"a string that I want to stuff into an object"
thedatasize = len(thedata) + 1
try:
# try using a null dkey
dkey = None
akey = b"this is the akey"
self.container.write_an_obj(thedata, thedatasize, dkey, akey, None,
None, 2)
self.plog.error("Didn't get expected return code.")
self.fail("Test was expected to return a -1003 but it has not.\n")
except DaosApiError as excep:
if '-1003' not in str(excep):
self.plog.error("Didn't get expected return code.")
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1003 but it has not.\n")
try:
# try using a null akey/io descriptor
dkey = b"this is the dkey"
akey = None
self.container.write_an_obj(thedata, thedatasize, dkey, akey, None,
None, 2)
self.fail("Test was expected to return a -1003 but it has not.\n")
except DaosApiError as excep:
if '-1003' not in str(excep):
self.plog.error("Didn't get expected return code.")
print(excep)
print(traceback.format_exc())
self.fail("Test was expected to get -1003 but it has not.\n")
try:
# lastly try passing no data
thedata = None
thedatasize = 0
dkey = b"this is the dkey"
akey = b"this is the akey"
self.container.write_an_obj(thedata, thedatasize, dkey, akey, None,
None, 2)
self.plog.info("Update with no data worked")
except DaosApiError as excep:
print(excep)
print(traceback.format_exc())
self.plog.error("Update with no data failed")
self.fail("Update with no data failed.\n")
self.plog.info("Test Complete")
