class DfuseCommand(ExecutableCommand):
"""Defines a object representing a dfuse command."""
def __init__(self, namespace, command):
"""Create a dfuse Command object."""
super(DfuseCommand, self).__init__(namespace, command)
# dfuse options
self.puuid = FormattedParameter("--pool {}")
self.cuuid = FormattedParameter("--container {}")
self.mount_dir = FormattedParameter("--mountpoint {}")
self.svcl = FormattedParameter("--svc {}", 0)
self.sys_name = FormattedParameter("--sys-name {}")
self.singlethreaded = FormattedParameter("--singlethreaded", False)
self.foreground = FormattedParameter("--foreground", False)
def set_dfuse_params(self, pool, display=True):
"""Set the dfuse parameters for the DAOS group, pool, and container uuid
Args:
pool (TestPool): DAOS test pool object
display (bool, optional): print updated params. Defaults to True.
"""
self.set_dfuse_pool_params(pool, display)
def set_dfuse_pool_params(self, pool, display=True):
"""Set Dfuse params based on Daos Pool.
Args:
pool (TestPool): DAOS test pool object
display (bool, optional): print updated params. Defaults to True.
"""
self.puuid.update(pool.uuid, "puuid" if display else None)
self.set_dfuse_svcl_param(pool, display)
def set_dfuse_svcl_param(self, pool, display=True):
"""Set the dfuse svcl param from the ranks of a DAOS pool object.
Args:
pool (TestPool): DAOS test pool object
display (bool, optional): print updated params. Defaults to True.
"""
svcl = ":".join(
[str(item) for item in [
int(pool.pool.svc.rl_ranks[index])
for index in range(pool.pool.svc.rl_nr)]])
self.svcl.update(svcl, "svcl" if display else None)
def set_dfuse_cont_param(self, cont, display=True):
"""Set dfuse cont param from Container object
Args:
cont (TestContainer): Daos test container object
display (bool, optional): print updated params. Defaults to True.
"""
self.cuuid.update(cont, "cuuid" if display else None)
class IorCommand(ExecutableCommand):
"""Defines a object for executing an IOR command.
Example:
>>> # Typical use inside of a DAOS avocado test method.
>>> ior_cmd = IorCommand()
>>> ior_cmd.get_params(self)
>>> ior_cmd.set_daos_params(self.server_group, self.pool)
>>> mpirun = Mpirun()
>>> env = self.ior_cmd.get_default_env(self.tmp, self.client_log)
>>> processes = len(self.hostlist_clients)
>>> mpirun.setup_command(env, self.hostfile_clients, processes)
>>> mpirun.run()
"""
def __init__(self):
"""Create an IorCommand object."""
super(IorCommand, self).__init__("/run/ior/*", "ior")
# Flags
self.flags = FormattedParameter("{}")
# Optional arguments
# -a=POSIX API for I/O [POSIX|DUMMY|MPIIO|MMAP|DAOS|DFS]
# -b=1048576 blockSize -- contiguous bytes to write per task
# -d=0 interTestDelay -- delay between reps in seconds
# -f=STRING scriptFile -- test script name
# -G=0 setTimeStampSignature -- time stamp signature
# -i=1 repetitions -- number of repetitions of test
# -j=0 outlierThreshold -- warn on outlier N sec from mean
# -J=1 setAlignment -- HDF5 alignment in bytes
# -l=STRING datapacket type-- type of packet created
# -M=STRING memoryPerNode -- hog memory on the node
# -N=0 numTasks -- num of participating tasks in the test
# -o=testFile testFile -- full name for test
# -O=STRING string of IOR directives
# -Q=1 taskPerNodeOffset for read tests
# -s=1 segmentCount -- number of segments
# -t=262144 transferSize -- size of transfer in bytes
# -T=0 maxTimeDuration -- max time in minutes executing
# repeated test; it aborts only between iterations
# and not within a test!
self.api = FormattedParameter("-a {}", "DAOS")
self.block_size = FormattedParameter("-b {}")
self.test_delay = FormattedParameter("-d {}")
self.script = FormattedParameter("-f {}")
self.signatute = FormattedParameter("-G {}")
self.repetitions = FormattedParameter("-i {}")
self.outlier_threshold = FormattedParameter("-j {}")
self.alignment = FormattedParameter("-J {}")
self.data_packet_type = FormattedParameter("-l {}")
self.memory_per_node = FormattedParameter("-M {}")
self.num_tasks = FormattedParameter("-N {}")
self.test_file = FormattedParameter("-o {}")
self.directives = FormattedParameter("-O {}")
self.task_offset = FormattedParameter("-Q {}")
self.segment_count = FormattedParameter("-s {}")
self.transfer_size = FormattedParameter("-t {}")
self.max_duration = FormattedParameter("-T {}")
# Module DAOS
# Required arguments
# --daos.pool=STRING pool uuid
# --daos.svcl=STRING pool SVCL
# --daos.cont=STRING container uuid
# Flags
# --daos.destroy Destroy Container
# Optional arguments
# --daos.group=STRING server group
# --daos.chunk_size=1048576 chunk size
# --daos.oclass=STRING object class
self.daos_pool = FormattedParameter("--daos.pool {}")
self.daos_svcl = FormattedParameter("--daos.svcl {}")
self.daos_cont = FormattedParameter("--daos.cont {}")
self.daos_destroy = FormattedParameter("--daos.destroy", True)
self.daos_group = FormattedParameter("--daos.group {}")
self.daos_chunk = FormattedParameter("--daos.chunk_size {}", 1048576)
self.daos_oclass = FormattedParameter("--daos.oclass {}", "SX")
def get_param_names(self):
"""Get a sorted list of the defined IorCommand parameters."""
# Sort the IOR parameter names to generate consistent ior commands
all_param_names = super(IorCommand, self).get_param_names()
# List all of the common ior params first followed by any daos-specific
# params (except when using MPIIO).
param_names = [name for name in all_param_names if "daos" not in name]
if self.api.value not in ["MPIIO", "POSIX"]:
param_names.extend(
[name for name in all_param_names if "daos" in name])
return param_names
def set_daos_params(self, group, pool, cont_uuid=None, display=True):
"""Set the IOR parameters for the DAOS group, pool, and container uuid.
Args:
group (str): DAOS server group name
pool (TestPool): DAOS test pool object
cont_uuid (str, optional): the container uuid. If not specified one
is generated. Defaults to None.
display (bool, optional): print updated params. Defaults to True.
"""
self.set_daos_pool_params(pool, display)
self.daos_group.update(group, "daos_group" if display else None)
self.daos_cont.update(cont_uuid if cont_uuid else uuid.uuid4(),
"daos_cont" if display else None)
def set_daos_pool_params(self, pool, display=True):
"""Set the IOR parameters that are based on a DAOS pool.
Args:
pool (TestPool): DAOS test pool object
display (bool, optional): print updated params. Defaults to True.
"""
self.daos_pool.update(pool.pool.get_uuid_str(),
"daos_pool" if display else None)
self.set_daos_svcl_param(pool, display)
def set_daos_svcl_param(self, pool, display=True):
"""Set the IOR daos_svcl param from the ranks of a DAOS pool object.
Args:
pool (TestPool): DAOS test pool object
display (bool, optional): print updated params. Defaults to True.
"""
svcl = ":".join([
str(item) for item in [
int(pool.pool.svc.rl_ranks[index])
for index in range(pool.pool.svc.rl_nr)
]
])
self.daos_svcl.update(svcl, "daos_svcl" if display else None)
def get_aggregate_total(self, processes):
"""Get the total bytes expected to be written by ior.
Args:
processes (int): number of processes running the ior command
Returns:
int: total number of bytes written
Raises:
CommandFailure: if there is an error obtaining the aggregate total
"""
power = {"k": 1, "m": 2, "g": 3, "t": 4}
total = processes
for name in ("block_size", "segment_count"):
item = getattr(self, name).value
if item:
sub_item = re.split(r"([^\d])", str(item))
if sub_item > 0:
total *= int(sub_item[0])
if len(sub_item) > 1:
key = sub_item[1].lower()
if key in power:
total *= 1024**power[key]
else:
raise CommandFailure(
"Error obtaining the IOR aggregate total from "
"the {} - bad key: value: {}, split: {}, "
"key: {}".format(name, item, sub_item, key))
else:
raise CommandFailure(
"Error obtaining the IOR aggregate total from the {}: "
"value: {}, split: {}".format(name, item, sub_item))
# Account for any replicas
try:
# Extract the replica quantity from the object class string
replica_qty = int(re.findall(r"\d+", self.daos_oclass.value)[0])
except (TypeError, IndexError):
# If the daos object class is undefined (TypeError) or it does not
# contain any numbers (IndexError) then there is only one replica
replica_qty = 1
finally:
total *= replica_qty
return total
def get_default_env(self, manager_cmd, attach_info, log_file=None):
"""Get the default enviroment settings for running IOR.
Args:
manager_cmd (str): job manager command
attach_info (str): CART attach info path
log_file (str, optional): log file. Defaults to None.
Returns:
EnvironmentVariables: a dictionary of environment names and values
"""
env = EnvironmentVariables()
env["CRT_ATTACH_INFO_PATH"] = attach_info
env["MPI_LIB"] = "\"\""
env["DAOS_SINGLETON_CLI"] = 1
env["FI_PSM2_DISCONNECT"] = 1
if log_file:
env["D_LOG_FILE"] = log_file
if "mpirun" in manager_cmd or "srun" in manager_cmd:
env["DAOS_POOL"] = self.daos_pool.value
env["DAOS_SVCL"] = self.daos_svcl.value
env["FI_PSM2_DISCONNECT"] = 1
env["IOR_HINT__MPI__romio_daos_obj_class"] = self.daos_oclass.value
return env
@staticmethod
def get_ior_metrics(cmdresult):
"""Parse the CmdResult (output of the test) and look for
the ior stdout and get the read and write metrics.
Args:
cmdresult (CmdResult): output of job manager
Returns:
metrics (tuple) : list of write and read metrics from ior run
"""
ior_metric_summary = "Summary of all tests:"
messages = cmdresult.stdout.splitlines()
# Get the index whre the summary starts and add one to
# get to the header.
idx = messages.index(ior_metric_summary)
# idx + 1 is header.
# idx +2 and idx + 3 will give the write and read metrics.
write_metrics = (" ".join(messages[idx + 2].split())).split()
read_metrics = (" ".join(messages[idx + 3].split())).split()
return (write_metrics, read_metrics)
@staticmethod
def log_metrics(logger, message, metrics):
"""Log the ior metrics
Args:
logger (log): logger object handle
message (str) : Message to print before logging metrics
metric (lst) : IOR write and read metrics
"""
logger.info("\n")
logger.info(message)
for m in metrics:
logger.info(m)
logger.info("\n")
class DmgCommand(YamlCommand):
"""Defines a object representing a dmg command."""
METHOD_REGEX = {
"run":
r"(.*)",
"network_scan":
r"(?:|[-]+\s+(.*)\s+[-]+(?:\n|\n\r))"
r"(?:.*\s+(fabric_iface|provider|pinned_numa_node):\s+"
r"([a-z0-9+;_]+))",
# Sample output of dmg pool list.
# wolf-3:10001: connected
# Pool UUID Svc Replicas
# --------- ------------
# b4a27b5b-688a-4d1e-8c38-363e32eb4f29 1,2,3
# Between the first and the second group, use " +"; i.e., one or more
# whitespaces. If we use "\s+", it'll pick up the second divider as
# UUID since it's made up of hyphens and \s includes new line.
"pool_list":
r"(?:([0-9a-fA-F-]+) +([0-9,]+))"
}
def __init__(self, path, yaml_cfg=None):
"""Create a dmg Command object.
Args:
path (str): path to the dmg command
yaml_cfg (DmgYamlParameters, optional): dmg config file
settings. Defaults to None, in which case settings
must be supplied as command-line paramters.
"""
super(DmgCommand, self).__init__("/run/dmg/*", "dmg", path, yaml_cfg)
# If specified use the configuration file from the YamlParameters object
default_yaml_file = None
if isinstance(self.yaml, YamlParameters):
default_yaml_file = self.yaml.filename
self._hostlist = FormattedParameter("-l {}")
self.hostfile = FormattedParameter("-f {}")
self.configpath = FormattedParameter("-o {}", default_yaml_file)
self.insecure = FormattedParameter("-i", False)
self.debug = FormattedParameter("-d", False)
self.json = FormattedParameter("-j", False)
@property
def hostlist(self):
"""Get the hostlist that was set.
Returns a string list.
"""
if self.yaml:
return self.yaml.hostlist.value
else:
return self._hostlist.value.split(",")
@hostlist.setter
def hostlist(self, hostlist):
"""Set the hostlist to be used for dmg invocation.
Args:
hostlist (string list): list of host addresses
"""
if self.yaml:
if not isinstance(hostlist, list):
hostlist = hostlist.split(",")
self.yaml.hostlist.update(hostlist, "dmg.yaml.hostlist")
else:
if isinstance(hostlist, list):
hostlist = ",".join(hostlist)
self._hostlist.update(hostlist, "dmg._hostlist")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg sub command object based upon the sub-command."""
if self.sub_command.value == "network":
self.sub_command_class = self.NetworkSubCommand()
elif self.sub_command.value == "pool":
self.sub_command_class = self.PoolSubCommand()
elif self.sub_command.value == "storage":
self.sub_command_class = self.StorageSubCommand()
elif self.sub_command.value == "system":
self.sub_command_class = self.SystemSubCommand()
else:
self.sub_command_class = None
class NetworkSubCommand(CommandWithSubCommand):
"""Defines an object for the dmg network sub command."""
def __init__(self):
"""Create a dmg network subcommand object."""
super(DmgCommand.NetworkSubCommand,
self).__init__("/run/dmg/network/*", "network")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg network sub command object."""
if self.sub_command.value == "scan":
self.sub_command_class = self.ScanSubCommand()
else:
self.sub_command_class = None
class ScanSubCommand(CommandWithParameters):
"""Defines an object for the dmg network scan command."""
def __init__(self):
"""Create a dmg network scan command object."""
super(DmgCommand.NetworkSubCommand.ScanSubCommand,
self).__init__("/run/dmg/network/scan/*", "scan")
self.provider = FormattedParameter("-p {}", None)
self.all = FormattedParameter("-a", False)
class PoolSubCommand(CommandWithSubCommand):
"""Defines an object for the dmg pool sub command."""
def __init__(self):
"""Create a dmg pool subcommand object."""
super(DmgCommand.PoolSubCommand,
self).__init__("/run/dmg/pool/*", "pool")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg pool sub command object."""
if self.sub_command.value == "create":
self.sub_command_class = self.CreateSubCommand()
elif self.sub_command.value == "delete-acl":
self.sub_command_class = self.DeleteAclSubCommand()
elif self.sub_command.value == "destroy":
self.sub_command_class = self.DestroySubCommand()
elif self.sub_command.value == "get-acl":
self.sub_command_class = self.GetAclSubCommand()
elif self.sub_command.value == "list":
self.sub_command_class = self.ListSubCommand()
elif self.sub_command.value == "overwrite-acl":
self.sub_command_class = self.OverwriteAclSubCommand()
elif self.sub_command.value == "query":
self.sub_command_class = self.QuerySubCommand()
elif self.sub_command.value == "set-prop":
self.sub_command_class = self.SetPropSubCommand()
elif self.sub_command.value == "update-acl":
self.sub_command_class = self.UpdateAclSubCommand()
else:
self.sub_command_class = None
class CreateSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool create command."""
def __init__(self):
"""Create a dmg pool create command object."""
super(DmgCommand.PoolSubCommand.CreateSubCommand,
self).__init__("/run/dmg/pool/create/*", "create")
self.group = FormattedParameter("--group={}", None)
self.user = FormattedParameter("--user={}", None)
self.acl_file = FormattedParameter("--acl-file={}", None)
self.scm_size = FormattedParameter("--scm-size={}", None)
self.nvme_size = FormattedParameter("--nvme-size={}", None)
self.ranks = FormattedParameter("--ranks={}", None)
self.nsvc = FormattedParameter("--nsvc={}", None)
self.sys = FormattedParameter("--sys={}", None)
class DeleteAclSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool delete-acl command."""
def __init__(self):
"""Create a dmg pool delete-acl command object."""
super(DmgCommand.PoolSubCommand.DeleteAclSubCommand,
self).__init__("/run/dmg/pool/delete-acl/*",
"delete-acl")
self.pool = FormattedParameter("--pool={}", None)
self.principal = FormattedParameter("-p {}", None)
class DestroySubCommand(CommandWithParameters):
"""Defines an object for the dmg pool destroy command."""
def __init__(self):
"""Create a dmg pool destroy command object."""
super(DmgCommand.PoolSubCommand.DestroySubCommand,
self).__init__("/run/dmg/pool/destroy/*", "destroy")
self.pool = FormattedParameter("--pool={}", None)
self.sys_name = FormattedParameter("--sys-name={}", None)
self.force = FormattedParameter("--force", False)
class GetAclSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool get-acl command."""
def __init__(self):
"""Create a dmg pool get-acl command object."""
super(DmgCommand.PoolSubCommand.GetAclSubCommand,
self).__init__("/run/dmg/pool/get-acl/*", "get-acl")
self.pool = FormattedParameter("--pool={}", None)
class ListSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool list command."""
def __init__(self):
"""Create a dmg pool list command object."""
super(DmgCommand.PoolSubCommand.ListSubCommand,
self).__init__("/run/dmg/pool/list/*", "list")
class OverwriteAclSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool overwrite-acl command."""
def __init__(self):
"""Create a dmg pool overwrite-acl command object."""
super(DmgCommand.PoolSubCommand.OverwriteAclSubCommand,
self).__init__("/run/dmg/pool/overwrite-acl/*",
"overwrite-acl")
self.pool = FormattedParameter("--pool={}", None)
self.acl_file = FormattedParameter("-a {}", None)
class QuerySubCommand(CommandWithParameters):
"""Defines an object for the dmg pool query command."""
def __init__(self):
"""Create a dmg pool query command object."""
super(DmgCommand.PoolSubCommand.QuerySubCommand,
self).__init__("/run/dmg/pool/query/*", "query")
self.pool = FormattedParameter("--pool={}", None)
class SetPropSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool set-prop command."""
def __init__(self):
"""Create a dmg pool set-prop command object."""
super(DmgCommand.PoolSubCommand.SetPropSubCommand,
self).__init__("/run/dmg/pool/set-prop/*", "set-prop")
self.pool = FormattedParameter("--pool={}", None)
self.name = FormattedParameter("--name={}", None)
self.value = FormattedParameter("--value={}", None)
class UpdateAclSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool update-acl command."""
def __init__(self):
"""Create a dmg pool update-acl command object."""
super(DmgCommand.PoolSubCommand.UpdateAclSubCommand,
self).__init__("/run/dmg/pool/update-acl/*",
"update-acl")
self.pool = FormattedParameter("--pool={}", None)
self.acl_file = FormattedParameter("-a {}", None)
self.entry = FormattedParameter("-e {}", None)
class StorageSubCommand(CommandWithSubCommand):
"""Defines an object for the dmg storage sub command."""
def __init__(self):
"""Create a dmg storage subcommand object."""
super(DmgCommand.StorageSubCommand,
self).__init__("/run/dmg/storage/*", "storage")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg storage sub command object."""
if self.sub_command.value == "format":
self.sub_command_class = self.FormatSubCommand()
elif self.sub_command.value == "prepare":
self.sub_command_class = self.PrepareSubCommand()
elif self.sub_command.value == "query":
self.sub_command_class = self.QuerySubCommand()
elif self.sub_command.value == "scan":
self.sub_command_class = self.ScanSubCommand()
elif self.sub_command.value == "set":
self.sub_command_class = self.SetSubCommand()
else:
self.sub_command_class = None
class FormatSubCommand(CommandWithParameters):
"""Defines an object for the dmg storage format command."""
def __init__(self):
"""Create a dmg storage format command object."""
super(DmgCommand.StorageSubCommand.FormatSubCommand,
self).__init__("/run/dmg/storage/format/*", "format")
self.reformat = FormattedParameter("--reformat", False)
class PrepareSubCommand(CommandWithParameters):
"""Defines an object for the dmg storage format command."""
def __init__(self):
"""Create a dmg storage prepare command object."""
super(DmgCommand.StorageSubCommand.PrepareSubCommand,
self).__init__("/run/dmg/storage/prepare/*", "prepare")
self.pci_whitelist = FormattedParameter("-w {}", None)
self.hugepages = FormattedParameter("-p {}", None)
self.target_user = FormattedParameter("-u {}", None)
self.nvme_only = FormattedParameter("-n", False)
self.scm_only = FormattedParameter("-s", False)
self.reset = FormattedParameter("--reset", False)
self.force = FormattedParameter("-f", False)
class QuerySubCommand(CommandWithSubCommand):
"""Defines an object for the dmg query format command."""
def __init__(self):
"""Create a dmg storage query command object."""
super(DmgCommand.StorageSubCommand.QuerySubCommand,
self).__init__("/run/dmg/storage/query/*", "query")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg pool sub command object."""
if self.sub_command.value == "blobstore-health":
self.sub_command_class = self.BlobstoreHealthSubCommand()
elif self.sub_command.value == "smd":
self.sub_command_class = self.SmdSubCommand()
else:
self.sub_command_class = None
class BlobstoreHealthSubCommand(CommandWithParameters):
"""Defines a dmg storage query blobstore-health object."""
def __init__(self):
"""Create a dmg storage query blobstore-health object."""
super(
DmgCommand.StorageSubCommand.QuerySubCommand.
BlobstoreHealthSubCommand, self).__init__(
"/run/dmg/storage/query/blobstore-health/*",
"blobstore-health")
self.devuuid = FormattedParameter("-u {}", None)
self.tgtid = FormattedParameter("-t {}", None)
class SmdSubCommand(CommandWithParameters):
"""Defines a dmg storage query smd object."""
def __init__(self):
"""Create a dmg storage query smd object."""
super(
DmgCommand.StorageSubCommand.QuerySubCommand.
SmdSubCommand,
self).__init__("/run/dmg/storage/query/smd/*", "smd")
self.devices = FormattedParameter("-d", False)
self.pools = FormattedParameter("-p", False)
class ScanSubCommand(CommandWithParameters):
"""Defines an object for the dmg storage scan command."""
def __init__(self):
"""Create a dmg storage scan command object."""
super(DmgCommand.StorageSubCommand.ScanSubCommand,
self).__init__("/run/dmg/storage/scan/*", "scan")
self.summary = FormattedParameter("-m", False)
class SetSubCommand(CommandWithParameters):
"""Defines an object for the dmg storage set command."""
def __init__(self):
"""Create a dmg storage set command object."""
super(DmgCommand.StorageSubCommand.SetSubCommand,
self).__init__("/run/dmg/storage/set/*", "set")
self.nvme_faulty = FormattedParameter("nvme-faulty", False)
class SystemSubCommand(CommandWithSubCommand):
"""Defines an object for the dmg system sub command."""
def __init__(self):
"""Create a dmg system subcommand object."""
super(DmgCommand.SystemSubCommand,
self).__init__("/run/dmg/system/*", "system")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg system sub command object."""
if self.sub_command.value == "leader-query":
self.sub_command_class = self.LeaderQuerySubCommand()
elif self.sub_command.value == "list-pools":
self.sub_command_class = self.ListPoolsSubCommand()
elif self.sub_command.value == "query":
self.sub_command_class = self.QuerySubCommand()
elif self.sub_command.value == "start":
self.sub_command_class = self.StartSubCommand()
elif self.sub_command.value == "stop":
self.sub_command_class = self.StopSubCommand()
else:
self.sub_command_class = None
class LeaderQuerySubCommand(CommandWithParameters):
"""Defines an object for the dmg system leader-query command."""
def __init__(self):
"""Create a dmg system leader-query command object."""
super(DmgCommand.SystemSubCommand.LeaderQuerySubCommand,
self).__init__("/run/dmg/system/leader-query/*",
"leader-query")
class ListPoolsSubCommand(CommandWithParameters):
"""Defines an object for the dmg system list-pools command."""
def __init__(self):
"""Create a dmg system list-pools command object."""
super(DmgCommand.SystemSubCommand.ListPoolsSubCommand,
self).__init__("/run/dmg/system/list-pools/*",
"list-pools")
class QuerySubCommand(CommandWithParameters):
"""Defines an object for the dmg system query command."""
def __init__(self):
"""Create a dmg system query command object."""
super(DmgCommand.SystemSubCommand.QuerySubCommand,
self).__init__("/run/dmg/system/query/*", "query")
self.rank = FormattedParameter("--rank={}")
self.verbose = FormattedParameter("--verbose", False)
class StartSubCommand(CommandWithParameters):
"""Defines an object for the dmg system start command."""
def __init__(self):
"""Create a dmg system start command object."""
super(DmgCommand.SystemSubCommand.StartSubCommand,
self).__init__("/run/dmg/system/start/*", "start")
class StopSubCommand(CommandWithParameters):
"""Defines an object for the dmg system stop command."""
def __init__(self):
"""Create a dmg system stop command object."""
super(DmgCommand.SystemSubCommand.StopSubCommand,
self).__init__("/run/dmg/system/stop/*", "stop")
self.force = FormattedParameter("--force", False)
def _get_result(self):
"""Get the result from running the configured dmg command.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg command fails.
"""
if self.yaml:
self.create_yaml_file()
result = None
try:
result = self.run()
except CommandFailure as error:
raise CommandFailure("<dmg> command failed: {}".format(error))
return result
def network_scan(self, provider=None, all_devs=False):
"""Get the result of the dmg network scan command.
Args:
provider (str): name of network provider tied to the device
all_devs (bool, optional): Show all devs info. Defaults to False.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg storage scan command fails.
"""
self.set_sub_command("network")
self.sub_command_class.set_sub_command("scan")
self.sub_command_class.sub_command_class.provider.value = provider
self.sub_command_class.sub_command_class.all.value = all_devs
return self._get_result()
def storage_scan(self):
"""Get the result of the dmg storage scan command.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg storage scan command fails.
"""
self.set_sub_command("storage")
self.sub_command_class.set_sub_command("scan")
return self._get_result()
def storage_format(self):
"""Get the result of the dmg storage format command.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg storage format command fails.
"""
self.set_sub_command("storage")
self.sub_command_class.set_sub_command("format")
return self._get_result()
def storage_prepare(self,
user=None,
hugepages="4096",
nvme=False,
scm=False,
reset=False,
force=True):
"""Get the result of the dmg storage format command.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg storage prepare command fails.
"""
self.set_sub_command("storage")
self.sub_command_class.set_sub_command("prepare")
self.sub_command_class.sub_command_class.nvme_only.value = nvme
self.sub_command_class.sub_command_class.scm_only.value = scm
self.sub_command_class.sub_command_class.target_user.value = \
getuser() if user is None else user
self.sub_command_class.sub_command_class.hugepages.value = hugepages
self.sub_command_class.sub_command_class.reset.value = reset
self.sub_command_class.sub_command_class.force.value = force
return self._get_result()
def pool_create(self,
scm_size,
uid=None,
gid=None,
nvme_size=None,
target_list=None,
svcn=None,
group=None,
acl_file=None):
"""Create a pool with the dmg command.
The uid and gid method arguments can be specified as either an integer
or a string. If an integer value is specified it will be converted into
the corresponding user/group name string.
Args:
scm_size (int): SCM pool size to create.
uid (object, optional): User ID with privileges. Defaults to None.
gid (object, otional): Group ID with privileges. Defaults to None.
nvme_size (str, optional): NVMe size. Defaults to None.
target_list (list, optional): a list of storage server unique
identifiers (ranks) for the DAOS pool
svcn (str, optional): Number of pool service replicas. Defaults to
None, in which case 1 is used by the dmg binary in default.
group (str, optional): DAOS system group name in which to create the
pool. Defaults to None, in which case "daos_server" is used by
default.
acl_file (str, optional): ACL file. Defaults to None.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg pool create command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("create")
self.sub_command_class.sub_command_class.user.value = \
getpwuid(uid).pw_name if isinstance(uid, int) else uid
self.sub_command_class.sub_command_class.group.value = \
getgrgid(gid).gr_name if isinstance(gid, int) else gid
self.sub_command_class.sub_command_class.scm_size.value = scm_size
self.sub_command_class.sub_command_class.nvme_size.value = nvme_size
if target_list is not None:
self.sub_command_class.sub_command_class.ranks.value = ",".join(
[str(target) for target in target_list])
self.sub_command_class.sub_command_class.nsvc.value = svcn
self.sub_command_class.sub_command_class.sys.value = group
self.sub_command_class.sub_command_class.acl_file.value = acl_file
return self._get_result()
def pool_destroy(self, pool, force=True):
"""Destroy a pool with the dmg command.
Args:
pool (str): Pool UUID to destroy.
force (bool, optional): Force removal of pool. Defaults to True.
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool destroy command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("destroy")
self.sub_command_class.sub_command_class.pool.value = pool
self.sub_command_class.sub_command_class.force.value = force
return self._get_result()
def pool_get_acl(self, pool):
"""Get the ACL for a given pool.
Args:
pool (str): Pool for which to get the ACL.
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool get-acl command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("get-acl")
self.sub_command_class.sub_command_class.pool.value = pool
return self._get_result()
def pool_update_acl(self, pool, acl_file, entry):
"""Update the acl for a given pool.
Args:
pool (str): Pool for which to update the ACL.
acl_file (str): ACL file to update
entry (str): entry to be updated
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool update-acl command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("update-acl")
self.sub_command_class.sub_command_class.pool.value = pool
self.sub_command_class.sub_command_class.acl_file.value = acl_file
self.sub_command_class.sub_command_class.entry.value = entry
return self._get_result()
def pool_overwrite_acl(self, pool, acl_file):
"""Overwrite the acl for a given pool.
Args:
pool (str): Pool for which to overwrite the ACL.
acl_file (str): ACL file to update
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool overwrite-acl command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("overwrite-acl")
self.sub_command_class.sub_command_class.pool.value = pool
self.sub_command_class.sub_command_class.acl_file.value = acl_file
return self._get_result()
def pool_delete_acl(self, pool, principal):
"""Delete the acl for a given pool.
Args:
pool (str): Pool for which to delete the ACL.
principal (str): principal to be deleted
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool delete-acl command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("delete-acl")
self.sub_command_class.sub_command_class.pool.value = pool
self.sub_command_class.sub_command_class.principal.value = principal
return self._get_result()
def pool_list(self):
"""List pools.
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool delete-acl command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("list")
return self._get_result()
def pool_set_prop(self, pool, name, value):
"""Set property for a given Pool.
Args:
pool (str): Pool uuid for which property is supposed
to be set.
name (str): Property name to be set
value (str): Property value to be set
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool set-prop command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("set-prop")
self.sub_command_class.sub_command_class.pool.value = pool
self.sub_command_class.sub_command_class.name.value = name
self.sub_command_class.sub_command_class.value.value = value
return self._get_result()
class DmgCommand(CommandWithSubCommand):
"""Defines a object representing a dmg command."""
def __init__(self, path):
"""Create a dmg Command object.
Args:
path (str): path to the dmg command
"""
super(DmgCommand, self).__init__("/run/dmg/*", "dmg", path)
self.hostlist = FormattedParameter("-l {}")
self.hostfile = FormattedParameter("-f {}")
self.configpath = FormattedParameter("-o {}")
self.insecure = FormattedParameter("-i", True)
self.debug = FormattedParameter("-d", False)
self.json = FormattedParameter("-j", False)
def set_hostlist(self, manager):
"""Set the dmg hostlist parameter with the daos server/agent info.
Use the daos server/agent access points port and list of hosts to define
the dmg --hostlist command line parameter.
Args:
manager (SubprocessManager): daos server/agent process manager
"""
self.hostlist.update(manager.get_config_value("access_points"),
"dmg.hostlist")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg sub command object based upon the sub-command."""
if self.sub_command.value == "network":
self.sub_command_class = self.NetworkSubCommand()
elif self.sub_command.value == "pool":
self.sub_command_class = self.PoolSubCommand()
elif self.sub_command.value == "storage":
self.sub_command_class = self.StorageSubCommand()
elif self.sub_command.value == "system":
self.sub_command_class = self.SystemSubCommand()
else:
self.sub_command_class = None
class NetworkSubCommand(CommandWithSubCommand):
"""Defines an object for the dmg network sub command."""
def __init__(self):
"""Create a dmg network subcommand object."""
super(DmgCommand.NetworkSubCommand,
self).__init__("/run/dmg/network/*", "network")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg network sub command object."""
if self.sub_command.value == "scan":
self.sub_command_class = self.ScanSubCommand()
else:
self.sub_command_class = None
class ScanSubCommand(CommandWithParameters):
"""Defines an object for the dmg network scan command."""
def __init__(self):
"""Create a dmg network scan command object."""
super(DmgCommand.NetworkSubCommand.ScanSubCommand,
self).__init__("/run/dmg/network/scan/*", "scan")
self.provider = FormattedParameter("-p {}", None)
self.all = FormattedParameter("-a", False)
class PoolSubCommand(CommandWithSubCommand):
"""Defines an object for the dmg pool sub command."""
def __init__(self):
"""Create a dmg pool subcommand object."""
super(DmgCommand.PoolSubCommand,
self).__init__("/run/dmg/pool/*", "pool")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg pool sub command object."""
if self.sub_command.value == "create":
self.sub_command_class = self.CreateSubCommand()
elif self.sub_command.value == "delete-acl":
self.sub_command_class = self.DeleteAclSubCommand()
elif self.sub_command.value == "destroy":
self.sub_command_class = self.DestroySubCommand()
elif self.sub_command.value == "get-acl":
self.sub_command_class = self.GetAclSubCommand()
elif self.sub_command.value == "list":
self.sub_command_class = self.ListSubCommand()
elif self.sub_command.value == "overwrite-acl":
self.sub_command_class = self.OverwriteAclSubCommand()
elif self.sub_command.value == "query":
self.sub_command_class = self.QuerySubCommand()
elif self.sub_command.value == "set-prop":
self.sub_command_class = self.SetPropSubCommand()
elif self.sub_command.value == "update-acl":
self.sub_command_class = self.UpdateAclSubCommand()
else:
self.sub_command_class = None
class CreateSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool create command."""
def __init__(self):
"""Create a dmg pool create command object."""
super(DmgCommand.PoolSubCommand.CreateSubCommand,
self).__init__("/run/dmg/pool/create/*", "create")
self.group = FormattedParameter("--group={}", None)
self.user = FormattedParameter("--user={}", None)
self.acl_file = FormattedParameter("--acl-file={}", None)
self.scm_size = FormattedParameter("--scm-size={}", None)
self.nvme_size = FormattedParameter("--nvme-size={}", None)
self.ranks = FormattedParameter("--ranks={}", None)
self.nsvc = FormattedParameter("--nsvc={}", None)
self.sys = FormattedParameter("--sys={}", None)
class DeleteAclSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool delete-acl command."""
def __init__(self):
"""Create a dmg pool delete-acl command object."""
super(DmgCommand.PoolSubCommand.DeleteAclSubCommand,
self).__init__("/run/dmg/pool/delete-acl/*",
"delete-acl")
self.pool = FormattedParameter("--pool={}", None)
self.principal = FormattedParameter("-p {}", None)
class DestroySubCommand(CommandWithParameters):
"""Defines an object for the dmg pool destroy command."""
def __init__(self):
"""Create a dmg pool destroy command object."""
super(DmgCommand.PoolSubCommand.DestroySubCommand,
self).__init__("/run/dmg/pool/destroy/*", "destroy")
self.pool = FormattedParameter("--pool={}", None)
self.sys_name = FormattedParameter("--sys-name={}", None)
self.force = FormattedParameter("--force", False)
class GetAclSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool get-acl command."""
def __init__(self):
"""Create a dmg pool get-acl command object."""
super(DmgCommand.PoolSubCommand.GetAclSubCommand,
self).__init__("/run/dmg/pool/get-acl/*", "get-acl")
self.pool = FormattedParameter("--pool={}", None)
class ListSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool list command."""
def __init__(self):
"""Create a dmg pool list command object."""
super(DmgCommand.PoolSubCommand.ListSubCommand,
self).__init__("/run/dmg/pool/list/*", "list")
class OverwriteAclSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool overwrite-acl command."""
def __init__(self):
"""Create a dmg pool overwrite-acl command object."""
super(DmgCommand.PoolSubCommand.OverwriteAclSubCommand,
self).__init__("/run/dmg/pool/overwrite-acl/*",
"overwrite-acl")
self.pool = FormattedParameter("--pool={}", None)
self.acl_file = FormattedParameter("-a {}", None)
class QuerySubCommand(CommandWithParameters):
"""Defines an object for the dmg pool query command."""
def __init__(self):
"""Create a dmg pool query command object."""
super(DmgCommand.PoolSubCommand.QuerySubCommand,
self).__init__("/run/dmg/pool/query/*", "query")
self.pool = FormattedParameter("--pool={}", None)
class SetPropSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool set-prop command."""
def __init__(self):
"""Create a dmg pool set-prop command object."""
super(DmgCommand.PoolSubCommand.SetPropSubCommand,
self).__init__("/run/dmg/pool/set-prop/*", "set-prop")
self.pool = FormattedParameter("--pool={}", None)
self.name = FormattedParameter("--name={}", None)
self.value = FormattedParameter("--value={}", None)
class UpdateAclSubCommand(CommandWithParameters):
"""Defines an object for the dmg pool update-acl command."""
def __init__(self):
"""Create a dmg pool update-acl command object."""
super(DmgCommand.PoolSubCommand.UpdateAclSubCommand,
self).__init__("/run/dmg/pool/update-acl/*",
"update-acl")
self.pool = FormattedParameter("--pool={}", None)
self.acl_file = FormattedParameter("-a {}", None)
self.entry = FormattedParameter("-e {}", None)
class StorageSubCommand(CommandWithSubCommand):
"""Defines an object for the dmg storage sub command."""
def __init__(self):
"""Create a dmg storage subcommand object."""
super(DmgCommand.StorageSubCommand,
self).__init__("/run/dmg/storage/*", "storage")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg storage sub command object."""
if self.sub_command.value == "format":
self.sub_command_class = self.FormatSubCommand()
elif self.sub_command.value == "prepare":
self.sub_command_class = self.PrepareSubCommand()
elif self.sub_command.value == "query":
self.sub_command_class = self.QuerySubCommand()
elif self.sub_command.value == "scan":
self.sub_command_class = self.ScanSubCommand()
elif self.sub_command.value == "set":
self.sub_command_class = self.SetSubCommand()
else:
self.sub_command_class = None
class FormatSubCommand(CommandWithParameters):
"""Defines an object for the dmg storage format command."""
def __init__(self):
"""Create a dmg storage format command object."""
super(DmgCommand.StorageSubCommand.FormatSubCommand,
self).__init__("/run/dmg/storage/format/*", "format")
self.reformat = FormattedParameter("--reformat", False)
class PrepareSubCommand(CommandWithParameters):
"""Defines an object for the dmg storage format command."""
def __init__(self):
"""Create a dmg storage prepare command object."""
super(DmgCommand.StorageSubCommand.PrepareSubCommand,
self).__init__("/run/dmg/storage/prepare/*", "prepare")
self.pci_whitelist = FormattedParameter("-w {}", None)
self.hugepages = FormattedParameter("-p {}", None)
self.target_user = FormattedParameter("-u {}", None)
self.nvme_only = FormattedParameter("-n", False)
self.scm_only = FormattedParameter("-s", False)
self.reset = FormattedParameter("--reset", False)
self.force = FormattedParameter("-f", False)
class QuerySubCommand(CommandWithSubCommand):
"""Defines an object for the dmg query format command."""
def __init__(self):
"""Create a dmg storage query command object."""
super(DmgCommand.StorageSubCommand.QuerySubCommand,
self).__init__("/run/dmg/storage/query/*", "query")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg pool sub command object."""
if self.sub_command.value == "blobstore-health":
self.sub_command_class = self.BlobstoreHealthSubCommand()
elif self.sub_command.value == "smd":
self.sub_command_class = self.SmdSubCommand()
else:
self.sub_command_class = None
class BlobstoreHealthSubCommand(CommandWithParameters):
"""Defines a dmg storage query blobstore-health object."""
def __init__(self):
"""Create a dmg storage query blobstore-health object."""
super(
DmgCommand.StorageSubCommand.QuerySubCommand.
BlobstoreHealthSubCommand, self).__init__(
"/run/dmg/storage/query/blobstore-health/*",
"blobstore-health")
self.devuuid = FormattedParameter("-u {}", None)
self.tgtid = FormattedParameter("-t {}", None)
class SmdSubCommand(CommandWithParameters):
"""Defines a dmg storage query smd object."""
def __init__(self):
"""Create a dmg storage query smd object."""
super(
DmgCommand.StorageSubCommand.QuerySubCommand.
SmdSubCommand,
self).__init__("/run/dmg/storage/query/smd/*", "smd")
self.devices = FormattedParameter("-d", False)
self.pools = FormattedParameter("-p", False)
class ScanSubCommand(CommandWithParameters):
"""Defines an object for the dmg storage scan command."""
def __init__(self):
"""Create a dmg storage scan command object."""
super(DmgCommand.StorageSubCommand.ScanSubCommand,
self).__init__("/run/dmg/storage/scan/*", "scan")
self.summary = FormattedParameter("-m", False)
class SetSubCommand(CommandWithParameters):
"""Defines an object for the dmg storage set command."""
def __init__(self):
"""Create a dmg storage set command object."""
super(DmgCommand.StorageSubCommand.SetSubCommand,
self).__init__("/run/dmg/storage/set/*", "set")
self.nvme_faulty = FormattedParameter("nvme-faulty", False)
class SystemSubCommand(CommandWithSubCommand):
"""Defines an object for the dmg system sub command."""
def __init__(self):
"""Create a dmg system subcommand object."""
super(DmgCommand.SystemSubCommand,
self).__init__("/run/dmg/system/*", "system")
def get_sub_command_class(self):
# pylint: disable=redefined-variable-type
"""Get the dmg system sub command object."""
if self.sub_command.value == "leader-query":
self.sub_command_class = self.LeaderQuerySubCommand()
elif self.sub_command.value == "list-pools":
self.sub_command_class = self.ListPoolsSubCommand()
elif self.sub_command.value == "query":
self.sub_command_class = self.QuerySubCommand()
elif self.sub_command.value == "start":
self.sub_command_class = self.StartSubCommand()
elif self.sub_command.value == "stop":
self.sub_command_class = self.StopSubCommand()
else:
self.sub_command_class = None
class LeaderQuerySubCommand(CommandWithParameters):
"""Defines an object for the dmg system leader-query command."""
def __init__(self):
"""Create a dmg system leader-query command object."""
super(DmgCommand.SystemSubCommand.LeaderQuerySubCommand,
self).__init__("/run/dmg/system/leader-query/*",
"leader-query")
class ListPoolsSubCommand(CommandWithParameters):
"""Defines an object for the dmg system list-pools command."""
def __init__(self):
"""Create a dmg system list-pools command object."""
super(DmgCommand.SystemSubCommand.ListPoolsSubCommand,
self).__init__("/run/dmg/system/list-pools/*",
"list-pools")
class QuerySubCommand(CommandWithParameters):
"""Defines an object for the dmg system query command."""
def __init__(self):
"""Create a dmg system query command object."""
super(DmgCommand.SystemSubCommand.QuerySubCommand,
self).__init__("/run/dmg/system/query/*", "query")
self.rank = FormattedParameter("--rank={}")
self.verbose = FormattedParameter("--verbose", False)
class StartSubCommand(CommandWithParameters):
"""Defines an object for the dmg system start command."""
def __init__(self):
"""Create a dmg system start command object."""
super(DmgCommand.SystemSubCommand.StartSubCommand,
self).__init__("/run/dmg/system/start/*", "start")
class StopSubCommand(CommandWithParameters):
"""Defines an object for the dmg system stop command."""
def __init__(self):
"""Create a dmg system stop command object."""
super(DmgCommand.SystemSubCommand.StopSubCommand,
self).__init__("/run/dmg/system/stop/*", "stop")
self.force = FormattedParameter("--force", False)
def _get_result(self):
"""Get the result from running the configured dmg command.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg command fails.
"""
result = None
try:
result = self.run()
except CommandFailure as error:
raise CommandFailure("<dmg> command failed: {}".format(error))
return result
def storage_scan(self):
"""Get the result of the dmg storage scan command.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg storage scan command fails.
"""
self.set_sub_command("storage")
self.sub_command_class.set_sub_command("scan")
return self._get_result()
def storage_format(self):
"""Get the result of the dmg storage format command.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg storage format command fails.
"""
self.set_sub_command("storage")
self.sub_command_class.set_sub_command("format")
return self._get_result()
def storage_prepare(self,
user=None,
hugepages="4096",
nvme=False,
scm=False,
reset=False,
force=True):
"""Get the result of the dmg storage format command.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg storage prepare command fails.
"""
self.set_sub_command("storage")
self.sub_command_class.set_sub_command("prepare")
self.sub_command_class.sub_command_class.nvme_only.value = nvme
self.sub_command_class.sub_command_class.scm_only.value = scm
self.sub_command_class.sub_command_class.target_user.value = \
getuser() if user is None else user
self.sub_command_class.sub_command_class.hugepages.value = hugepages
self.sub_command_class.sub_command_class.reset.value = reset
self.sub_command_class.sub_command_class.force.value = force
return self._get_result()
def pool_create(self,
scm_size,
uid=None,
gid=None,
nvme_size=None,
target_list=None,
svcn=None,
group=None,
acl_file=None):
"""Create a pool with the dmg command.
The uid and gid method arguments can be specified as either an integer
or a string. If an integer value is specified it will be converted into
the corresponding user/group name string.
Args:
scm_size (int): SCM pool size to create.
uid (object, optional): User ID with privileges. Defaults to None.
gid (object, otional): Group ID with privileges. Defaults to None.
nvme_size (str, optional): NVMe size. Defaults to None.
target_list (list, optional): a list of storage server unique
identifiers (ranks) for the DAOS pool
svcn (str, optional): Number of pool service replicas. Defaults to
None, in which case 1 is used by the dmg binary in default.
group (str, optional): DAOS system group name in which to create the
pool. Defaults to None, in which case "daos_server" is used by
default.
acl_file (str, optional): ACL file. Defaults to None.
Returns:
CmdResult: an avocado CmdResult object containing the dmg command
information, e.g. exit status, stdout, stderr, etc.
Raises:
CommandFailure: if the dmg pool create command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("create")
self.sub_command_class.sub_command_class.user.value = \
getpwuid(uid).pw_name if isinstance(uid, int) else uid
self.sub_command_class.sub_command_class.group.value = \
getgrgid(gid).gr_name if isinstance(gid, int) else gid
self.sub_command_class.sub_command_class.scm_size.value = scm_size
self.sub_command_class.sub_command_class.nvme_size.value = nvme_size
if target_list is not None:
self.sub_command_class.sub_command_class.ranks.value = ",".join(
[str(target) for target in target_list])
self.sub_command_class.sub_command_class.nsvc.value = svcn
self.sub_command_class.sub_command_class.sys.value = group
self.sub_command_class.sub_command_class.acl_file.value = acl_file
return self._get_result()
def pool_destroy(self, pool, force=True):
"""Destroy a pool with the dmg command.
Args:
pool (str): Pool UUID to destroy.
force (bool, optional): Force removal of pool. Defaults to True.
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool destroy command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("destroy")
self.sub_command_class.sub_command_class.pool.value = pool
self.sub_command_class.sub_command_class.force.value = force
return self._get_result()
def pool_get_acl(self, pool):
"""Get the ACL for a given pool.
Args:
pool (str): Pool for which to get the ACL.
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool get-acl command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("get-acl")
self.sub_command_class.sub_command_class.pool.value = pool
return self._get_result()
def pool_update_acl(self, pool, acl_file, entry):
"""Update the acl for a given pool.
Args:
pool (str): Pool for which to update the ACL.
acl_file (str): ACL file to update
entry (str): entry to be updated
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool update-acl command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("update-acl")
self.sub_command_class.sub_command_class.pool.value = pool
self.sub_command_class.sub_command_class.acl_file.value = acl_file
self.sub_command_class.sub_command_class.entry.value = entry
return self._get_result()
def pool_overwrite_acl(self, pool, acl_file):
"""Overwrite the acl for a given pool.
Args:
pool (str): Pool for which to overwrite the ACL.
acl_file (str): ACL file to update
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool overwrite-acl command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("overwrite-acl")
self.sub_command_class.sub_command_class.pool.value = pool
self.sub_command_class.sub_command_class.acl_file.value = acl_file
return self._get_result()
def pool_delete_acl(self, pool, principal):
"""Delete the acl for a given pool.
Args:
pool (str): Pool for which to delete the ACL.
principal (str): principal to be deleted
Returns:
CmdResult: Object that contains exit status, stdout, and other
information.
Raises:
CommandFailure: if the dmg pool delete-acl command fails.
"""
self.set_sub_command("pool")
self.sub_command_class.set_sub_command("delete-acl")
self.sub_command_class.sub_command_class.pool.value = pool
self.sub_command_class.sub_command_class.principal.value = principal
return self._get_result()
class MdtestCommand(ExecutableCommand):
"""Defines a object representing a mdtest command."""
def __init__(self):
"""Create an MdtestCommand object."""
super(MdtestCommand, self).__init__("/run/mdtest/*", "mdtest")
self.flags = FormattedParameter("{}") # mdtest flags
# Optional arguments
# -a=STRING API for I/O [POSIX|DUMMY]
# -b=1 branching factor of hierarchical dir structure
# -d=./out the directory in which the tests will run
# -B=0 no barriers between phases
# -e=0 bytes to read from each file
# -f=1 first number of tasks on which test will run
# -i=1 number of iterations the test will run
# -I=0 number of items per directory in tree
# -l=0 last number of tasks on which test will run
# -n=0 every process will creat/stat/read/remove num
# of directories and files
# -N=0 stride num between neighbor tasks for file/dir
# operation (local=0)
# -p=0 pre-iteration delay (in seconds)
# --random-seed=0 random seed for -R
# -s=1 stride between number of tasks for each test
# -V=0 verbosity value
# -w=0 bytes to write each file after it is created
# -W=0 number in seconds; stonewall timer, write as
# many seconds and ensure all processes did the
# same number of operations (currently only
# stops during create phase)
# -x=STRING StoneWallingStatusFile; contains the number
# of iterations of the creation phase, can be
# used to split phases across runs
# -z=0 depth of hierarchical directory structure
self.api = FormattedParameter("-a {}")
self.branching_factor = FormattedParameter("-b {}")
self.test_dir = FormattedParameter("-d {}")
self.barriers = FormattedParameter("-B {}")
self.read_bytes = FormattedParameter("-e {}")
self.first_num_tasks = FormattedParameter("-f {}")
self.iteration = FormattedParameter("-i {}")
self.items = FormattedParameter("-I {}")
self.last_num_tasks = FormattedParameter("-l {}")
self.num_of_files_dirs = FormattedParameter("-n {}")
self.pre_iter = FormattedParameter("-p {}")
self.random_seed = FormattedParameter("--random-seed {}")
self.stride = FormattedParameter("-s {}")
self.verbosity_value = FormattedParameter("-V {}")
self.write_bytes = FormattedParameter("-w {}")
self.stonewall_timer = FormattedParameter("-W {}")
self.stonewall_statusfile = FormattedParameter("-x {}")
self.depth = FormattedParameter("-z {}")
# Module DAOS (Not intended to be used as of now, hence all
# arguments for DAOS module are commented)
# Required arguments
# --daos.pool=STRING pool uuid
# --daos.svcl=STRING pool SVCL
# --daos.cont=STRING container uuid
# Flags
# --daos.destroy Destroy Container
# Optional arguments
# --daos.group=STRING server group
# --daos.chunk_size=1048576 chunk size
# --daos.oclass=STRING object class
# self.daos_pool_uuid = FormattedParameter("--daos.pool {}")
# self.daos_svcl = FormattedParameter("--daos.svcl {}")
# self.daos_cont = FormattedParameter("--daos.cont {}")
# self.daos_group = FormattedParameter("--daos.group {}")
# self.daos_chunk_size = FormattedParameter(" --daos.chunk_size {}")
# self.daos_oclass = FormattedParameter("--daos.oclass {}")
# self.daos_destroy = FormattedParameter("--daos.destroy", True)
# Module DFS
# Required arguments
# --dfs.pool=STRING DAOS pool uuid
# --dfs.svcl=STRING DAOS pool SVCL
# --dfs.cont=STRING DFS container uuid
# Flags
# --dfs.destroy Destroy DFS Container
# Optional arguments
# --dfs.group=STRING DAOS server group
self.dfs_pool_uuid = FormattedParameter("--dfs.pool {}")
self.dfs_svcl = FormattedParameter("--dfs.svcl {}")
self.dfs_cont = FormattedParameter("--dfs.cont {}")
self.dfs_group = FormattedParameter("--dfs.group {}")
self.dfs_destroy = FormattedParameter("--dfs.destroy", True)
def get_param_names(self):
"""Get a sorted list of the defined MdtestCommand parameters."""
# Sort the Mdtest parameter names to generate consistent ior commands
all_param_names = super(MdtestCommand, self).get_param_names()
# List all of the common ior params first followed by any dfs-specific
# params (except when using POSIX).
param_names = [name for name in all_param_names if "dfs" not in name]
if self.api.value != "POSIX":
param_names.extend(
[name for name in all_param_names if "dfs" in name])
return param_names
def set_daos_params(self, group, pool, cont_uuid=None, display=True):
"""Set the Mdtest params for the DAOS group, pool, and container uuid.
Args:
group (str): DAOS server group name
pool (TestPool): DAOS test pool object
cont_uuid (str, optional): the container uuid. If not specified one
is generated. Defaults to None.
display (bool, optional): print updated params. Defaults to True.
"""
self.set_daos_pool_params(pool, display)
self.dfs_group.update(group, "dfs_group" if display else None)
self.dfs_cont.update(
cont_uuid if cont_uuid else uuid.uuid4(),
"dfs_cont" if display else None)
def set_daos_pool_params(self, pool, display=True):
"""Set the Mdtest parameters that are based on a DAOS pool.
Args:
pool (TestPool): DAOS test pool object
display (bool, optional): print updated params. Defaults to True.
"""
self.dfs_pool_uuid.update(
pool.pool.get_uuid_str(), "dfs_pool" if display else None)
self.set_daos_svcl_param(pool, display)
def set_daos_svcl_param(self, pool, display=True):
"""Set the Mdtest daos_svcl param from the ranks of a DAOS pool object.
Args:
pool (TestPool): DAOS test pool object
display (bool, optional): print updated params. Defaults to True.
"""
svcl = ":".join(
[str(item) for item in [
int(pool.pool.svc.rl_ranks[index])
for index in range(pool.pool.svc.rl_nr)]])
self.dfs_svcl.update(svcl, "dfs_svcl" if display else None)
def get_default_env(self, manager_cmd, attach_info, log_file=None):
"""Get the default enviroment settings for running mdtest.
Args:
manager_cmd (str): job manager command
attach_info (str): CART attach info path
log_file (str, optional): log file. Defaults to None.
Returns:
EnvironmentVariables: a dictionary of environment names and values
"""
env = EnvironmentVariables()
env["CRT_ATTACH_INFO_PATH"] = attach_info
env["MPI_LIB"] = "\"\""
env["DAOS_SINGLETON_CLI"] = 1
env["FI_PSM2_DISCONNECT"] = 1
if log_file:
env["D_LOG_FILE"] = log_file
if "mpirun" in manager_cmd or "srun" in manager_cmd:
env["DAOS_POOL"] = self.dfs_pool_uuid.value
env["DAOS_SVCL"] = self.dfs_svcl.value
env["FI_PSM2_DISCONNECT"] = 1
return env
class IorCommand(CommandWithParameters):
"""Defines a object for executing an IOR command.
Example:
>>> # Typical use inside of a DAOS avocado test method.
>>> ior_cmd = IorCommand()
>>> ior_cmd.get_params(self)
>>> ior_cmd.set_daos_params(self.server_group, self.pool)
>>> ior_cmd.run(
self.basepath, len(self.hostlist_clients),
self.hostfile_clients)
"""
def __init__(self):
"""Create an IorCommand object."""
super(IorCommand, self).__init__("ior")
# Flags
self.flags = FormattedParameter("{}")
# Optional arguments
# -a=POSIX API for I/O [POSIX|DUMMY|MPIIO|MMAP|DAOS|DFS]
# -b=1048576 blockSize -- contiguous bytes to write per task
# -d=0 interTestDelay -- delay between reps in seconds
# -f=STRING scriptFile -- test script name
# -G=0 setTimeStampSignature -- time stamp signature
# -i=1 repetitions -- number of repetitions of test
# -j=0 outlierThreshold -- warn on outlier N sec from mean
# -J=1 setAlignment -- HDF5 alignment in bytes
# -l=STRING datapacket type-- type of packet created
# -M=STRING memoryPerNode -- hog memory on the node
# -N=0 numTasks -- num of participating tasks in the test
# -o=testFile testFile -- full name for test
# -O=STRING string of IOR directives
# -Q=1 taskPerNodeOffset for read tests
# -s=1 segmentCount -- number of segments
# -t=262144 transferSize -- size of transfer in bytes
# -T=0 maxTimeDuration -- max time in minutes executing
# repeated test; it aborts only between iterations
# and not within a test!
self.api = FormattedParameter("-a {}", "DAOS")
self.block_size = FormattedParameter("-b {}")
self.test_delay = FormattedParameter("-d {}")
self.script = FormattedParameter("-f {}")
self.signatute = FormattedParameter("-G {}")
self.repetitions = FormattedParameter("-i {}")
self.outlier_threshold = FormattedParameter("-j {}")
self.alignment = FormattedParameter("-J {}")
self.data_packet_type = FormattedParameter("-l {}")
self.memory_per_node = FormattedParameter("-M {}")
self.num_tasks = FormattedParameter("-N {}")
self.test_file = FormattedParameter("-o {}")
self.directives = FormattedParameter("-O {}")
self.task_offset = FormattedParameter("-Q {}")
self.segment_count = FormattedParameter("-s {}")
self.transfer_size = FormattedParameter("-t {}")
self.max_duration = FormattedParameter("-T {}")
# Module DAOS
# Required arguments
# --daos.pool=STRING pool uuid
# --daos.svcl=STRING pool SVCL
# --daos.cont=STRING container uuid
# Flags
# --daos.destroy Destroy Container
# Optional arguments
# --daos.group=STRING server group
# --daos.chunk_size=1048576 chunk size
# --daos.oclass=STRING object class
self.daos_pool = FormattedParameter("--daos.pool {}")
self.daos_svcl = FormattedParameter("--daos.svcl {}")
self.daos_cont = FormattedParameter("--daos.cont {}")
self.daos_destroy = FormattedParameter("--daos.destroy", True)
self.daos_group = FormattedParameter("--daos.group {}")
self.daos_chunk = FormattedParameter("--daos.chunk_size {}", 1048576)
self.daos_oclass = FormattedParameter("--daos.oclass {}")
def get_param_names(self):
"""Get a sorted list of the defined IorCommand parameters."""
# Sort the IOR parameter names to generate consistent ior commands
all_param_names = super(IorCommand, self).get_param_names()
# List all of the common ior params first followed by any daos-specific
# params (except when using MPIIO).
param_names = [name for name in all_param_names if "daos" not in name]
if self.api.value != "MPIIO":
param_names.extend(
[name for name in all_param_names if "daos" in name])
return param_names
def get_params(self, test, path="/run/ior/*"):
"""Get values for all of the ior command params using a yaml file.
Sets each BasicParameter object's value to the yaml key that matches
the assigned name of the BasicParameter object in this class. For
example, the self.block_size.value will be set to the value in the yaml
file with the key 'block_size'.
Args:
test (Test): avocado Test object
path (str, optional): yaml namespace. Defaults to "/run/ior/*".
"""
super(IorCommand, self).get_params(test, path)
def set_daos_params(self,
group,
pool,
cont_uuid=None,
display=True,
mpiio_oclass=None):
"""Set the IOR parameters for the DAOS group, pool, and container uuid.
Args:
group (str): DAOS server group name
pool (DaosPool): DAOS pool API object
cont_uuid (str, optional): the container uuid. If not specified one
is generated. Defaults to None.
display (bool, optional): print updated params. Defaults to True.
"""
self.set_daos_pool_params(pool, display)
self.daos_group.update(group, "daos_group" if display else None)
self.daos_cont.update(cont_uuid if cont_uuid else uuid.uuid4(),
"daos_cont" if display else None)
# assigning obj class as SX in None else
# the desired one
if mpiio_oclass is None:
self.mpiio_oclass = 214
else:
self.mpiio_oclass = mpiio_oclass
def set_daos_pool_params(self, pool, display=True):
"""Set the IOR parameters that are based on a DAOS pool.
Args:
pool (DaosPool): DAOS pool API object
display (bool, optional): print updated params. Defaults to True.
"""
#self.daos_pool.value = pool.uuid
self.daos_pool.update(pool.pool.get_uuid_str(),
"daos_pool" if display else None)
self.set_daos_svcl_param(pool, display)
def set_daos_svcl_param(self, pool, display=True):
"""Set the IOR daos_svcl param from the ranks of a DAOS pool object.
Args:
pool (DaosPool): DAOS pool API object
display (bool, optional): print updated params. Defaults to True.
"""
svcl = ":".join([
str(item) for item in [
int(pool.pool.svc.rl_ranks[index])
for index in range(pool.pool.svc.rl_nr)
]
])
self.daos_svcl.update(svcl, "daos_svcl" if display else None)
def get_aggregate_total(self, processes):
"""Get the total bytes expected to be written by ior.
Args:
processes (int): number of processes running the ior command
Returns:
int: total number of bytes written
"""
power = {"k": 1, "m": 2, "g": 3, "t": 4}
total = processes
for name in ("block_size", "segment_count"):
item = getattr(self, name).value
if item:
sub_item = re.split(r"([^\d])", str(item))
if sub_item > 0:
total *= int(sub_item[0])
if len(sub_item) > 1:
key = sub_item[1].lower()
if key in power:
total *= 1024**power[key]
else:
raise IorFailed(
"Error obtaining the IOR aggregate total from "
"the {} - bad key: value: {}, split: {}, "
"key: {}".format(name, item, sub_item, key))
else:
raise IorFailed(
"Error obtaining the IOR aggregate total from the {}: "
"value: {}, split: {}".format(name, item, sub_item))
# Account for any replicas
try:
# Extract the replica quantity from the object class string
replica_qty = int(re.findall(r"\d+", self.daos_oclass.value)[0])
except (TypeError, IndexError):
# If the daos object class is undefined (TypeError) or it does not
# contain any numbers (IndexError) then there is only one replica
replica_qty = 1
finally:
total *= replica_qty
return total
def get_launch_command(self, manager, attach_info, processes, hostfile):
"""Get the process launch command used to run IOR.
Args:
manager (str): mpi job manager command
attach_info (str): CART attach info path
mpi_prefix (str): path for the mpi launch command
processes (int): number of host processes
hostfile (str): file defining host names and slots
Raises:
IorFailed: if an error occured building the IOR command
Returns:
str: ior launch command
"""
print("Getting launch command for {}".format(manager))
exports = ""
env = {
"CRT_ATTACH_INFO_PATH": attach_info,
"MPI_LIB": "\"\"",
"DAOS_SINGLETON_CLI": 1,
}
if manager.endswith("mpirun"):
env.update({
"DAOS_POOL":
self.daos_pool.value,
"DAOS_SVCL":
self.daos_svcl.value,
"FI_PSM2_DISCONNECT":
1,
"IOR_HINT__MPI__romio_daos_obj_class":
self.mpiio_oclass,
})
assign_env = ["{}={}".format(key, val) for key, val in env.items()]
exports = "export {}; ".format("; export ".join(assign_env))
args = [
"-np {}".format(processes),
"-hostfile {}".format(hostfile),
# "-map-by node",
]
elif manager.endswith("orterun"):
assign_env = ["{}={}".format(key, val) for key, val in env.items()]
args = [
"-np {}".format(processes),
"-hostfile {}".format(hostfile),
"-map-by node",
]
args.extend(["-x {}".format(assign) for assign in assign_env])
elif manager.endswith("srun"):
env.update({
"DAOS_POOL": self.daos_pool.value,
"DAOS_SVCL": self.daos_svcl.value,
"FI_PSM2_DISCONNECT": 1,
})
assign_env = ["{}={}".format(key, val) for key, val in env.items()]
args = [
"-l",
"--mpi=pmi2",
"--export={}".format(",".join(["ALL"] + assign_env)),
]
if processes is not None:
args.append("--ntasks={}".format(processes))
args.append("--distribution=cyclic") # --map-by node
if hostfile is not None:
args.append("--nodefile={}".format(hostfile))
else:
raise IorFailed("Unsupported job manager: {}".format(manager))
return "{}{} {} {}".format(exports, manager, " ".join(args),
self.__str__())
def run(self, manager, attach_info, processes, hostfile, display=True):
"""Run the IOR command.
Args:
manager (str): mpi job manager command
attach_info (str): CART attach info path
processes (int): number of host processes
hostfile (str): file defining host names and slots
display (bool, optional): print IOR output to the console.
Defaults to True.
Raises:
IorFailed: if an error occured runnig the IOR command
"""
command = self.get_launch_command(manager, attach_info, processes,
hostfile)
if display:
print("<IOR CMD>: {}".format(command))
# Run IOR
try:
run(command, allow_output_check="combined", shell=True)
except CmdError as error:
print("<IorRunFailed> Exception occurred: {}".format(error))
raise IorFailed("IOR Run process Failed: {}".format(error))
