def __init__(self, pool, cb_handler=None):
"""Create a TeestContainer object.
Args:
pool (TestPool): the test pool in which to create the container
cb_handler (CallbackHandler, optional): callback object to use with
the API methods. Defaults to None.
"""
super(TestContainer, self).__init__("/run/container/*", cb_handler)
self.pool = pool
self.object_qty = BasicParameter(None)
self.record_qty = BasicParameter(None)
self.akey_size = BasicParameter(None)
self.dkey_size = BasicParameter(None)
self.data_size = BasicParameter(None)
self.data_array_size = BasicParameter(0, 0)
# Provider access to get input params values
# for enabling different container properties
self.input_params = DaosInputParams()
self.container = None
self.uuid = None
self.info = None
self.opened = False
self.written_data = []
def __init__(self, pool, cb_handler=None, daos_command=None):
"""Create a TeestContainer object.
Args:
pool (TestPool): the test pool in which to create the container
cb_handler (CallbackHandler, optional): callback object to use with
the API methods. Defaults to None.
"""
super(TestContainer, self).__init__("/run/container/*", cb_handler)
self.pool = pool
self.object_qty = BasicParameter(None)
self.record_qty = BasicParameter(None)
self.akey_size = BasicParameter(None)
self.dkey_size = BasicParameter(None)
self.data_size = BasicParameter(None)
self.data_array_size = BasicParameter(0, 0)
# Provider access to get input params values
# for enabling different container properties
self.input_params = DaosInputParams()
# Optional daos command object to use with the USE_DAOS control method
self.daos = daos_command
# Optional daos command argument values to use with the USE_DAOS control
# method when creating/destroying containers
self.path = BasicParameter(None)
self.type = BasicParameter(None)
self.oclass = BasicParameter(None)
self.chunk_size = BasicParameter(None)
self.properties = BasicParameter(None)
self.daos_timeout = BasicParameter(None)
self.container = None
self.uuid = None
self.info = None
self.opened = False
self.written_data = []
self.epoch = None
class TestContainer(TestDaosApiBase):
"""A class for functional testing of DaosContainer objects."""
def __init__(self, pool, cb_handler=None, daos_command=None):
"""Create a TeestContainer object.
Args:
pool (TestPool): the test pool in which to create the container
cb_handler (CallbackHandler, optional): callback object to use with
the API methods. Defaults to None.
"""
super(TestContainer, self).__init__("/run/container/*", cb_handler)
self.pool = pool
self.object_qty = BasicParameter(None)
self.record_qty = BasicParameter(None)
self.akey_size = BasicParameter(None)
self.dkey_size = BasicParameter(None)
self.data_size = BasicParameter(None)
self.data_array_size = BasicParameter(0, 0)
# Provider access to get input params values
# for enabling different container properties
self.input_params = DaosInputParams()
# Optional daos command object to use with the USE_DAOS control method
self.daos = daos_command
# Optional daos command argument values to use with the USE_DAOS control
# method when creating/destroying containers
self.path = BasicParameter(None)
self.type = BasicParameter(None)
self.oclass = BasicParameter(None)
self.chunk_size = BasicParameter(None)
self.properties = BasicParameter(None)
self.daos_timeout = BasicParameter(None)
self.container = None
self.uuid = None
self.info = None
self.opened = False
self.written_data = []
self.epoch = None
def __str__(self):
"""Return a string representation of this TestContainer object.
Returns:
str: the container's UUID, if defined
"""
if self.container is not None and self.uuid is not None:
return str(self.uuid)
return super(TestContainer, self).__str__()
def get_params(self, test):
"""Get values for all of the command params from the yaml file.
Sets each BasicParameter object's value to the yaml key that matches
the assigned name of the BasicParameter object in this class. For
example, the self.block_size.value will be set to the value in the yaml
file with the key 'block_size'.
If no key matches are found in the yaml file the BasicParameter object
will be set to its default value.
Args:
test (Test): avocado Test object
"""
super(TestContainer, self).get_params(test)
if self.daos:
self.daos.timeout = self.daos_timeout.value
@fail_on(DaosApiError)
@fail_on(CommandFailure)
def create(self, uuid=None, con_in=None, acl_file=None):
"""Create a container.
Args:
uuid (str, optional): container uuid. Defaults to None.
con_in (optional): to be defined. Defaults to None.
acl_file (str, optional): path of the ACL file. Defaults to None.
"""
self.destroy()
self.log.info("Creating a container with pool handle %s",
self.pool.pool.handle.value)
self.container = DaosContainer(self.pool.context)
if self.control_method.value == self.USE_API:
# Create a container with the API method
kwargs = {"poh": self.pool.pool.handle}
if uuid is not None:
kwargs["con_uuid"] = uuid
# Refer daos_api for setting input params for DaosContainer.
if con_in is not None:
cop = self.input_params.get_con_create_params()
cop.type = con_in[0]
cop.enable_chksum = con_in[1]
cop.srv_verify = con_in[2]
cop.chksum_type = con_in[3]
cop.chunk_size = con_in[4]
kwargs["con_prop"] = cop
self._call_method(self.container.create, kwargs)
elif self.control_method.value == self.USE_DAOS and self.daos:
# Disconnect the pool if connected
self.pool.disconnect()
# Create a container with the daos command
kwargs = {
"pool": self.pool.uuid,
"sys_name": self.pool.name.value,
"cont": uuid,
"path": self.path.value,
"cont_type": self.type.value,
"oclass": self.oclass.value,
"chunk_size": self.chunk_size.value,
"properties": self.properties.value,
"acl_file": acl_file,
}
self._log_method("daos.container_create", kwargs)
uuid = self.daos.get_output("container_create", **kwargs)[0]
# Populte the empty DaosContainer object with the properties of the
# container created with daos container create.
self.container.uuid = str_to_c_uuid(uuid)
self.container.attached = 1
self.container.poh = self.pool.pool.handle
elif self.control_method.value == self.USE_DAOS:
self.log.error("Error: Undefined daos command")
else:
self.log.error("Error: Undefined control_method: %s",
self.control_method.value)
self.uuid = self.container.get_uuid_str()
self.log.info(" Container created with uuid %s", self.uuid)
@fail_on(DaosApiError)
@fail_on(CommandFailure)
def create_snap(self, snap_name=None, epoch=None):
"""Create Snapshot using daos utility.
Args:
snap_name (str, optional): Snapshot name. Defaults to None.
epoch (str, optional): Epoch ID. Defaults to None.
"""
self.log.info("Creating Snapshot for Container: %s", self.uuid)
if self.control_method.value == self.USE_DAOS and self.daos:
# create snapshot using daos utility
kwargs = {
"pool": self.pool.uuid,
"cont": self.uuid,
"snap_name": snap_name,
"epoch": epoch,
"sys_name": self.pool.name.value
}
self._log_method("daos.container_create_snap", kwargs)
data = self.daos.container_create_snap(**kwargs)
elif self.control_method.value == self.USE_DAOS:
self.log.error("Error: Undefined daos command")
else:
self.log.error("Error: Undefined control_method: %s",
self.control_method.value)
self.epoch = data["epoch"]
@fail_on(DaosApiError)
@fail_on(CommandFailure)
def destroy_snap(self, snap_name=None, epc=None, epcrange=None):
"""Destroy Snapshot using daos utility.
Args:
snap_name (str, optional): Snapshot name
epc (str, optional): Epoch ID that indicates the snapshot to be
destroyed. Defaults to None.
epcrange (str, optional): Epoch range in the format "<start>-<end>".
"""
status = False
self.log.info("Destroying Snapshot for Container: %s", self.uuid)
if self.control_method.value == self.USE_DAOS and self.daos:
# destroy snapshot using daos utility
kwargs = {
"pool": self.pool.uuid,
"cont": self.uuid,
"snap_name": snap_name,
"epc": epc,
"epcrange": epcrange,
"sys_name": self.pool.name.value,
}
self._log_method("daos.container_destroy_snap", kwargs)
self.daos.container_destroy_snap(**kwargs)
status = True
elif self.control_method.value == self.USE_DAOS:
self.log.error("Error: Undefined daos command")
else:
self.log.error("Error: Undefined control_method: %s",
self.control_method.value)
self.epoch = None
return status
@fail_on(DaosApiError)
def open(self, pool_handle=None, container_uuid=None):
"""Open the container with pool handle and container UUID if provided.
Args:
pool_handle (TestPool.pool.handle, optional): Pool handle.
Defaults to None.
If you don't provide it, the default pool handle in
DaosContainer will be used.
If you created a TestPool instance and want to use its pool
handle, pass in something like self.pool[-1].pool.handle.value
container_uuid (hex, optional): Container UUID. Defaults to None.
If you want to use certain container's UUID, pass in
something like uuid.UUID(self.container[-1].uuid)
Returns:
bool: True if the container has been opened; False if the container
is already opened.
"""
if self.container and not self.opened:
self.log.info("Opening container %s", self.uuid)
self.pool.connect()
kwargs = {}
kwargs["poh"] = pool_handle
kwargs["cuuid"] = container_uuid
self._call_method(self.container.open, kwargs)
self.opened = True
return True
return False
@fail_on(DaosApiError)
def close(self):
"""Close the container.
Returns:
bool: True if the container has been closed; False if the container
is already closed.
"""
if self.container and self.opened:
self.log.info("Closing container %s", self.uuid)
self._call_method(self.container.close, {})
self.opened = False
return True
return False
@fail_on(DaosApiError)
@fail_on(CommandFailure)
def destroy(self, force=1):
"""Destroy the container.
Args:
force (int, optional): force flag. Defaults to 1.
Returns:
bool: True if the container has been destroyed; False if the
container does not exist.
"""
status = False
if self.container:
self.close()
self.log.info("Destroying container %s", self.uuid)
if self.container.attached:
kwargs = {"force": force}
if self.control_method.value == self.USE_API:
# Destroy the container with the API method
self._call_method(self.container.destroy, kwargs)
status = True
elif self.control_method.value == self.USE_DAOS and self.daos:
# Disconnect the pool if connected
self.pool.disconnect()
# Destroy the container with the daos command
kwargs["pool"] = self.pool.uuid
kwargs["sys_name"] = self.pool.name.value
kwargs["cont"] = self.uuid
self._log_method("daos.container_destroy", kwargs)
self.daos.container_destroy(**kwargs)
status = True
elif self.control_method.value == self.USE_DAOS:
self.log.error("Error: Undefined daos command")
else:
self.log.error("Error: Undefined control_method: %s",
self.control_method.value)
self.container = None
self.uuid = None
self.info = None
self.written_data = []
return status
@fail_on(DaosApiError)
def get_info(self, coh=None):
"""Query the container for information.
Sets the self.info attribute.
Args:
coh (str, optional): container handle override. Defaults to None.
"""
if self.container:
self.open()
self.log.info("Querying container %s", self.uuid)
self._call_method(self.container.query, {"coh": coh})
self.info = self.container.info
def check_container_info(self, ci_uuid=None, ci_nsnapshots=None):
# pylint: disable=unused-argument
"""Check the container info attributes.
Note:
Arguments may also be provided as a string with a number preceded
by '<', '<=', '>', or '>=' for other comparisons besides the
default '=='.
Args:
ci_uuid (str, optional): container uuid. Defaults to None.
ci_nsnapshots (int, optional): number of snapshots.
Defaults to None.
Note:
Arguments may also be provided as a string with a number preceded
by '<', '<=', '>', or '>=' for other comparisons besides the
default '=='.
Returns:
bool: True if at least one expected value is specified and all the
specified values match; False otherwise
"""
self.get_info()
checks = [(key, c_uuid_to_str(getattr(self.info, key))
if key == "ci_uuid" else getattr(self.info, key), val)
for key, val in locals().items()
if key != "self" and val is not None]
return self._check_info(checks)
@fail_on(DaosTestError)
def write_objects(self, rank=None, obj_class=None):
"""Write objects to the container.
Args:
rank (int, optional): server rank. Defaults to None.
obj_class (int, optional): daos object class. Defaults to None.
Raises:
DaosTestError: if there was an error writing the object
"""
self.open()
self.log.info(
"Writing %s object(s), with %s record(s) of %s bytes(s) each, in "
"container %s%s%s", self.object_qty.value, self.record_qty.value,
self.data_size.value, self.uuid,
" on rank {}".format(rank) if rank is not None else "",
" with object class {}".format(obj_class)
if obj_class is not None else "")
for _ in range(self.object_qty.value):
self.written_data.append(TestContainerData(self.debug.value))
self.written_data[-1].write_object(self, self.record_qty.value,
self.akey_size.value,
self.dkey_size.value,
self.data_size.value, rank,
obj_class,
self.data_array_size.value)
@fail_on(DaosTestError)
def read_objects(self, txn=None):
"""Read the objects from the container and verify they match.
Args:
txn (int, optional): transaction timestamp to read. Defaults to None
which uses the last timestamp written.
Returns:
bool: True if all the container objects contain the same previously
written records; False otherwise
"""
self.open()
self.log.info("Reading %s object(s) in container %s",
len(self.written_data), self.uuid)
status = len(self.written_data) > 0
for data in self.written_data:
data.debug = self.debug.value
status &= data.read_object(self, txn)
return status
def execute_io(self, duration, rank=None, obj_class=None):
"""Execute writes and reads for the specified time period.
Args:
duration (int): how long, in seconds, to write and read data
rank (int, optional): server rank. Defaults to None.
obj_class (int, optional): daos object class. Defaults to None.
Returns:
int: number of bytes written to the container
Raises:
DaosTestError: if there is an error writing, reading, or verify the
data
"""
self.open()
self.log.info(
"Writing and reading objects in container %s for %s seconds",
self.uuid, duration)
total_bytes_written = 0
finish_time = time() + duration
while time() < finish_time:
self.written_data.append(TestContainerData(self.debug.value))
self.written_data[-1].write_object(self, 1, self.akey_size.value,
self.dkey_size.value,
self.data_size.value, rank,
obj_class)
total_bytes_written += self.data_size.value
return total_bytes_written
def get_target_rank_lists(self, message=""):
"""Get a list of lists of target ranks from each written object.
Args:
message (str, optional): message to include in the target rank list
output. Defaults to "".
Raises:
DaosTestError: if there is an error obtaining the target rank list
from the DaosObj
Returns:
list: a list of list of targets for each written object in this
container
"""
self.open()
target_rank_lists = []
for data in self.written_data:
try:
data.obj.get_layout()
# Convert the list of longs into a list of ints
target_rank_lists.append(
[int(rank) for rank in data.obj.tgt_rank_list])
except DaosApiError as error:
raise DaosTestError(
"Error obtaining target rank list for object {} in "
"container {}: {}".format(data.obj, self.uuid, error))
if message is not None:
self.log.info("Target rank lists%s:", message)
for ranks in target_rank_lists:
self.log.info(" %s", ranks)
return target_rank_lists
def get_target_rank_count(self, rank, target_rank_list):
"""Get the number of times a rank appears in the target rank list.
Args:
rank (int): the rank to count. Defaults to None.
target_rank_list (list): a list of lists of target ranks per object
Returns:
(int): the number of object rank lists containing the rank
"""
count = sum([ranks.count(rank) for ranks in target_rank_list])
self.log.info("Occurrences of rank %s in the target rank list: %s",
rank, count)
return count
def punch_objects(self, indices):
"""Punch committed objects from the container.
Args:
indices (list): list of index numbers indicating which written
objects to punch from the self.written_data list
Raises:
DaosTestError: if there is an error punching the object or
determining which object to punch
Returns:
int: number of successfully punched objects
"""
self.open()
self.log.info(
"Punching %s objects from container %s with %s written objects",
len(indices), self.uuid, len(self.written_data))
count = 0
if self.written_data:
for index in indices:
# Find the object to punch at the specified index
txn = 0
try:
obj = self.written_data[index].obj
except IndexError:
raise DaosTestError(
"Invalid index {} for written data".format(index))
# Close the object
self.log.info(
"Closing object %s (index: %s, txn: %s) in container %s",
obj, index, txn, self.uuid)
try:
self._call_method(obj.close, {})
except DaosApiError:
continue
# Punch the object
self.log.info(
"Punching object %s (index: %s, txn: %s) from container %s",
obj, index, txn, self.uuid)
try:
self._call_method(obj.punch, {"txn": txn})
count += 1
except DaosApiError:
continue
# Mark the object's records as punched
for record in self.written_data[index].records:
record["punched"] = True
# Retutrn the number of punched objects
return count
def punch_records(self, indices, punch_dkey=True):
"""Punch committed records from the container.
Args:
indices (list): list of index numbers indicating which written
records in the object to punch from the self.written_data list
punch_dkey (bool, optional): whether to punch dkeys or akeys.
Defaults to True (punch dkeys).
Raises:
DaosTestError: if there is an error punching the record or
determining which record to punch
Returns:
int: number of successfully punched records
"""
self.open()
self.log.info(
"Punching %s records from each object in container %s with %s "
"written objects", len(indices), self.uuid, len(self.written_data))
count = 0
for data in self.written_data:
# Close the object
self.log.info("Closing object %s in container %s", data.obj,
self.uuid)
try:
self._call_method(data.obj.close, {})
except DaosApiError:
continue
# Find the record to punch at the specified index
for index in indices:
try:
rec = data.records[index]
except IndexError:
raise DaosTestError(
"Invalid record index {} for object {}".format(
index, data.obj))
# Punch the record
self.log.info(
"Punching record %s (index: %s, akey: %s, dkey: %s) from "
"object %s in container %s", rec, index, rec["akey"],
rec["dkey"], data.obj, self.uuid)
kwargs = {"txn": 0}
try:
if punch_dkey:
kwargs["dkeys"] = [rec["dkey"]]
self._call_method(data.obj.punch_dkeys, kwargs)
else:
kwargs["dkey"] = rec["dkey"]
kwargs["akeys"] = [rec["akey"]]
self._call_method(data.obj.punch_akeys, kwargs)
count += 1
except DaosApiError:
continue
# Mark the record as punched
rec["punched"] = True
# Return the number of punched objects
return count
