def setUp(self):
super(ObjectDataValidation, self).setUp()
self.obj = None
self.ioreq = None
self.no_of_dkeys = None
self.no_of_akeys = None
self.array_size = None
self.record_length = None
self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0]
self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0]
self.array_size = self.params.get("size", '/array_size/')
self.record_length = self.params.get("length", '/run/record/*')
self.prepare_pool()
self.container = DaosContainer(self.context)
self.container.create(self.pool.pool.handle)
self.container.open()
self.obj = DaosObj(self.context, self.container)
self.obj.create(objcls=1)
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj,
objtype=4)
def setUp(self):
super().setUp()
self.agent_sessions = None
self.pool = None
self.container = None
self.obj = None
self.ioreq = None
self.no_of_dkeys = None
self.no_of_akeys = None
self.array_size = None
self.record_length = None
self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0]
self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0]
self.record_length = self.params.get("length", '/run/record/*')
self.add_pool(connect=False)
self.pool.connect(2)
self.csum = self.params.get("enable_checksum", '/run/container/*')
self.container = DaosContainer(self.context)
input_param = self.container.cont_input_values
input_param.enable_chksum = self.csum
self.container.create(poh=self.pool.pool.handle,
con_prop=input_param)
self.container.open()
self.obj = DaosObj(self.context, self.container)
self.obj.create(objcls=1)
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj, objtype=4)
def write_single_object(self):
"""Write some data to the existing pool.
"""
self.pool.connect(2)
csum = self.params.get("enable_checksum", '/run/container/*')
container = DaosContainer(self.context)
input_param = container.cont_input_values
input_param.enable_chksum = csum
container.create(poh=self.pool.pool.handle,
con_prop=input_param)
container.open()
obj = DaosObj(self.context, container)
obj.create(objcls=1)
obj.open()
ioreq = IORequest(self.context,
container,
obj, objtype=4)
self.log.info("Writing the Single Dataset")
record_index = 0
for dkey in range(self.no_of_dkeys):
for akey in range(self.no_of_akeys):
indata = ("{0}".format(str(akey)[0])
* self.record_length[record_index])
d_key_value = "dkey {0}".format(dkey)
c_dkey = ctypes.create_string_buffer(d_key_value)
a_key_value = "akey {0}".format(akey)
c_akey = ctypes.create_string_buffer(a_key_value)
c_value = ctypes.create_string_buffer(indata)
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
ioreq.single_insert(c_dkey, c_akey, c_value, c_size)
record_index = record_index + 1
if record_index == len(self.record_length):
record_index = 0
def create_dataset(self, cont):
"""Create the dataset.
Args:
cont (TestContainer): the container
"""
self.log.info("Creating dataset in %s/%s", str(cont.pool.uuid),
str(cont.uuid))
cont.open()
for obj_idx in range(self.num_objs):
# Create a new obj
obj = DaosObj(cont.pool.context, cont.container)
self.obj_list.append(obj)
obj.create(rank=obj_idx, objcls=2)
obj.open()
ioreq = IORequest(cont.pool.context, cont.container, obj)
for dkey_idx in range(self.num_dkeys_per_obj):
c_dkey = ctypes.create_string_buffer(
"dkey {}".format(dkey_idx).encode())
for akey_idx in range(self.num_akeys_single_per_dkey):
# Round-robin to get the size of data and
# arbitrarily use a number 0-9 to fill data
akey_size_idx = akey_idx % len(self.akey_sizes)
data_size = self.akey_sizes[akey_size_idx]
data_val = str(akey_idx % 10)
data = data_size * data_val
c_akey = ctypes.create_string_buffer(
"akey single {}".format(akey_idx).encode())
c_value = ctypes.create_string_buffer(data.encode())
c_size = ctypes.c_size_t(ctypes.sizeof(c_value))
ioreq.single_insert(c_dkey, c_akey, c_value, c_size)
for akey_idx in range(self.num_akeys_array_per_dkey):
# Round-robin to get the size of data and
# the number of extents, and
# arbitrarily use a number 0-9 to fill data
akey_size_idx = akey_idx % len(self.akey_sizes)
data_size = self.akey_sizes[akey_size_idx]
akey_extent_idx = akey_idx % len(self.akey_extents)
num_extents = self.akey_extents[akey_extent_idx]
c_data = []
akey = "akey array {}".format(akey_idx)
c_akey = ctypes.create_string_buffer(akey.encode())
for data_idx in range(num_extents):
data_val = str(data_idx % 10)
data = data_size * data_val
c_data.append([
ctypes.create_string_buffer(data.encode()),
data_size
])
ioreq.insert_array(c_dkey, c_akey, c_data)
obj.close()
cont.close()
def setUp(self):
self.agent_sessions = None
self.pool = None
self.container = None
self.obj = None
self.ioreq = None
self.hostlist = None
self.hostfile = None
self.no_of_dkeys = None
self.no_of_akeys = None
self.array_size = None
self.record_length = None
with open('../../.build_vars.json') as json_f:
build_paths = json.load(json_f)
self.basepath = os.path.normpath(build_paths['PREFIX'] + "/../")
server_group = self.params.get("name", '/server_config/',
'daos_server')
self.context = DaosContext(build_paths['PREFIX'] + '/lib64/')
self.d_log = DaosLog(self.context)
self.hostlist = self.params.get("test_machines", '/run/hosts/*')
self.hostfile = write_host_file.write_host_file(
self.hostlist, self.workdir)
self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0]
self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0]
self.array_size = self.params.get("size", '/array_size/')
self.record_length = self.params.get("length", '/run/record/*')
self.agent_sessions = agent_utils.run_agent(self.basepath,
self.hostlist)
server_utils.run_server(self, self.hostfile, server_group)
self.pool = DaosPool(self.context)
self.pool.create(self.params.get("mode", '/run/pool/createmode/*'),
os.geteuid(), os.getegid(),
self.params.get("size", '/run/pool/createsize/*'),
self.params.get("setname", '/run/pool/createset/*'),
None)
self.pool.connect(2)
self.container = DaosContainer(self.context)
self.container.create(self.pool.handle)
self.container.open()
self.obj = DaosObj(self.context, self.container)
self.obj.create(objcls=1)
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj,
objtype=4)
def setUp(self):
super(ObjectDataValidation, self).setUp()
self.agent_sessions = None
self.pool = None
self.container = None
self.obj = None
self.ioreq = None
self.hostlist = None
self.hostfile = None
self.no_of_dkeys = None
self.no_of_akeys = None
self.array_size = None
self.record_length = None
server_group = self.params.get("name", '/server_config/',
'daos_server')
self.hostlist = self.params.get("test_servers", '/run/hosts/*')
self.hostfile = write_host_file.write_host_file(
self.hostlist, self.workdir)
self.no_of_dkeys = self.params.get("no_of_dkeys", '/run/dkeys/*')[0]
self.no_of_akeys = self.params.get("no_of_akeys", '/run/akeys/*')[0]
self.array_size = self.params.get("size", '/array_size/')
self.record_length = self.params.get("length", '/run/record/*')
self.agent_sessions = agent_utils.run_agent(self, self.hostlist)
server_utils.run_server(self, self.hostfile, server_group)
self.pool = DaosPool(self.context)
self.pool.create(self.params.get("mode", '/run/pool/createmode/*'),
os.geteuid(), os.getegid(),
self.params.get("size", '/run/pool/createsize/*'),
self.params.get("setname", '/run/pool/createset/*'),
None)
self.pool.connect(2)
self.container = DaosContainer(self.context)
self.container.create(self.pool.handle)
self.container.open()
self.obj = DaosObj(self.context, self.container)
self.obj.create(objcls=1)
self.obj.open()
self.ioreq = IORequest(self.context,
self.container,
self.obj,
objtype=4)
def dataset_verify(self, obj_list, cont, num_objs, num_dkeys,
num_akeys_single, num_akeys_array, akey_sizes,
akey_extents):
"""Verify a dataset generated with dataset_gen.
Args:
obj_list (list): obj_list returned from dataset_gen.
cont (TestContainer): the container.
num_objs (int): number of objects created in the container.
num_dkeys (int): number of dkeys created per object.
num_akeys_single (int): number of DAOS_IOD_SINGLE akeys per dkey.
num_akeys_array (int): number of DAOS_IOD_ARRAY akeys per dkey.
akey_sizes (list): varying akey sizes to iterate.
akey_extents (list): varying number of akey extents to iterate.
"""
self.log.info("Verifying dataset in %s/%s", str(cont.pool.uuid),
str(cont.uuid))
cont.open()
for obj_idx in range(num_objs):
# Open the obj
c_oid = obj_list[obj_idx].c_oid
obj = DaosObj(cont.pool.context, cont.container, c_oid=c_oid)
obj.open()
ioreq = IORequest(cont.pool.context, cont.container, obj)
for dkey_idx in range(num_dkeys):
dkey = "dkey {}".format(dkey_idx)
c_dkey = create_string_buffer(dkey)
for akey_idx in range(num_akeys_single):
# Round-robin to get the size of data and
# arbitrarily use a number 0-9 to fill data
akey_size_idx = akey_idx % len(akey_sizes)
data_size = akey_sizes[akey_size_idx]
data_val = str(akey_idx % 10)
data = data_size * data_val
akey = "akey single {}".format(akey_idx)
c_akey = create_string_buffer(akey)
c_data = ioreq.single_fetch(c_dkey, c_akey, data_size + 1)
actual_data = str(c_data.value.decode())
if actual_data != data:
self.log.info("Expected:\n%s\nBut got:\n%s",
data[:100] + "...",
actual_data[:100] + "...")
self.log.info("For:\nobj: %s.%s\ndkey: %s\nakey: %s",
str(obj.c_oid.hi), str(obj.c_oid.lo),
dkey, akey)
self.fail("Single value verification failed.")
for akey_idx in range(num_akeys_array):
# Round-robin to get the size of data and
# the number of extents, and
# arbitrarily use a number 0-9 to fill data
akey_size_idx = akey_idx % len(akey_sizes)
data_size = akey_sizes[akey_size_idx]
akey_extent_idx = akey_idx % len(akey_extents)
num_extents = akey_extents[akey_extent_idx]
akey = "akey array {}".format(akey_idx)
c_akey = create_string_buffer(akey)
c_num_extents = ctypes.c_uint(num_extents)
c_data_size = ctypes.c_size_t(data_size)
actual_data = ioreq.fetch_array(c_dkey, c_akey,
c_num_extents, c_data_size)
for data_idx in range(num_extents):
data_val = str(data_idx % 10)
data = data_size * data_val
actual_idx = str(actual_data[data_idx].decode())
if data != actual_idx:
self.log.info("Expected:\n%s\nBut got:\n%s",
data[:100] + "...",
actual_idx + "...")
self.log.info(
"For:\nobj: %s.%s\ndkey: %s\nakey: %s",
str(obj.c_oid.hi), str(obj.c_oid.lo), dkey,
akey)
self.fail("Array verification failed.")
obj.close()
cont.close()
def dataset_gen(self, cont, num_objs, num_dkeys, num_akeys_single,
num_akeys_array, akey_sizes, akey_extents):
"""Generate a dataset with some number of objects, dkeys, and akeys.
Expects the container to be created with the API control method.
Args:
cont (TestContainer): the container.
num_objs (int): number of objects to create in the container.
num_dkeys (int): number of dkeys to create per object.
num_akeys_single (int): number of DAOS_IOD_SINGLE akeys per dkey.
num_akeys_array (int): number of DAOS_IOD_ARRAY akeys per dkey.
akey_sizes (list): varying akey sizes to iterate.
akey_extents (list): varying number of akey extents to iterate.
Returns:
list: a list of DaosObj created.
"""
self.log.info("Creating dataset in %s/%s", str(cont.pool.uuid),
str(cont.uuid))
cont.open()
obj_list = []
for obj_idx in range(num_objs):
# Open the obj
obj = DaosObj(cont.pool.context, cont.container)
obj_list.append(obj)
obj.create(rank=obj_idx, objcls=2)
obj.open()
ioreq = IORequest(cont.pool.context, cont.container, obj)
for dkey_idx in range(num_dkeys):
dkey = "dkey {}".format(dkey_idx)
c_dkey = create_string_buffer(dkey)
for akey_idx in range(num_akeys_single):
# Round-robin to get the size of data and
# arbitrarily use a number 0-9 to fill data
akey_size_idx = akey_idx % len(akey_sizes)
data_size = akey_sizes[akey_size_idx]
data_val = str(akey_idx % 10)
data = data_size * data_val
akey = "akey single {}".format(akey_idx)
c_akey = create_string_buffer(akey)
c_data = create_string_buffer(data)
c_size = ctypes.c_size_t(ctypes.sizeof(c_data))
ioreq.single_insert(c_dkey, c_akey, c_data, c_size)
for akey_idx in range(num_akeys_array):
# Round-robin to get the size of data and
# the number of extents, and
# arbitrarily use a number 0-9 to fill data
akey_size_idx = akey_idx % len(akey_sizes)
data_size = akey_sizes[akey_size_idx]
akey_extent_idx = akey_idx % len(akey_extents)
num_extents = akey_extents[akey_extent_idx]
akey = "akey array {}".format(akey_idx)
c_akey = create_string_buffer(akey)
c_data = []
for data_idx in range(num_extents):
data_val = str(data_idx % 10)
data = data_size * data_val
c_data.append([create_string_buffer(data), data_size])
ioreq.insert_array(c_dkey, c_akey, c_data)
obj.close()
cont.close()
return obj_list
def verify_dataset(self, cont):
"""Verify the dataset.
Args:
cont (TestContainer): the container
"""
self.log.info("Verifying dataset in %s/%s", str(cont.pool.uuid),
str(cont.uuid))
cont.open()
for obj_idx in range(self.num_objs):
obj = DaosObj(cont.pool.context, cont.container,
self.obj_list[obj_idx].c_oid)
obj.open()
ioreq = IORequest(cont.pool.context, cont.container, obj)
for dkey_idx in range(self.num_dkeys_per_obj):
dkey = "dkey {}".format(dkey_idx)
c_dkey = ctypes.create_string_buffer(dkey.encode())
for akey_idx in range(self.num_akeys_single_per_dkey):
# Round-robin to get the size of data and
# arbitrarily use a number 0-9 to fill data
akey_size_idx = akey_idx % len(self.akey_sizes)
data_size = self.akey_sizes[akey_size_idx]
data_val = str(akey_idx % 10)
data = str(data_size * data_val)
akey = "akey single {}".format(akey_idx)
c_akey = ctypes.create_string_buffer(akey.encode())
buf = ioreq.single_fetch(c_dkey, c_akey, data_size + 1)
actual_data = str(buf.value.decode())
if actual_data != data:
self.log.info("Expected:\n%s\nBut got:\n%s",
data[:100] + "...",
actual_data[:100] + "...")
self.log.info("For:\nobj: %s.%s\ndkey: %s\nakey: %s",
str(obj.c_oid.hi), str(obj.c_oid.lo),
dkey, akey)
self.fail("Single value verification failed.")
for akey_idx in range(self.num_akeys_array_per_dkey):
# Round-robin to get the size of data and
# the number of extents, and
# arbitrarily use a number 0-9 to fill data
akey_size_idx = akey_idx % len(self.akey_sizes)
data_size = self.akey_sizes[akey_size_idx]
c_data_size = ctypes.c_size_t(data_size)
akey_extent_idx = akey_idx % len(self.akey_extents)
num_extents = self.akey_extents[akey_extent_idx]
c_num_extents = ctypes.c_uint(num_extents)
akey = "akey array {}".format(akey_idx)
c_akey = ctypes.create_string_buffer(akey.encode())
actual_data = ioreq.fetch_array(c_dkey, c_akey,
c_num_extents, c_data_size)
for data_idx in range(num_extents):
data_val = str(data_idx % 10)
data = str(data_size * data_val)
actual_idx = str(actual_data[data_idx].decode())
if data != actual_idx:
self.log.info("Expected:\n%s\nBut got:\n%s",
data[:100] + "...",
actual_idx + "...")
self.log.info(
"For:\nobj: %s.%s\ndkey: %s\nakey: %s",
str(obj.c_oid.hi), str(obj.c_oid.lo), dkey,
akey)
self.fail("Array verification failed.")
obj.close()
cont.close()