class DfuseCommand(ExecutableCommand): """Defines a object representing a dfuse command.""" def __init__(self, namespace, command): """Create a dfuse Command object.""" super(DfuseCommand, self).__init__(namespace, command) # dfuse options self.puuid = FormattedParameter("--pool {}") self.cuuid = FormattedParameter("--container {}") self.mount_dir = FormattedParameter("--mountpoint {}") self.svcl = FormattedParameter("--svc {}", 0) self.sys_name = FormattedParameter("--sys-name {}") self.singlethreaded = FormattedParameter("--singlethreaded", False) self.foreground = FormattedParameter("--foreground", False) def set_dfuse_params(self, pool, display=True): """Set the dfuse parameters for the DAOS group, pool, and container uuid Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ self.set_dfuse_pool_params(pool, display) def set_dfuse_pool_params(self, pool, display=True): """Set Dfuse params based on Daos Pool. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ self.puuid.update(pool.uuid, "puuid" if display else None) self.set_dfuse_svcl_param(pool, display) def set_dfuse_svcl_param(self, pool, display=True): """Set the dfuse svcl param from the ranks of a DAOS pool object. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ svcl = ":".join( [str(item) for item in [ int(pool.pool.svc.rl_ranks[index]) for index in range(pool.pool.svc.rl_nr)]]) self.svcl.update(svcl, "svcl" if display else None) def set_dfuse_cont_param(self, cont, display=True): """Set dfuse cont param from Container object Args: cont (TestContainer): Daos test container object display (bool, optional): print updated params. Defaults to True. """ self.cuuid.update(cont, "cuuid" if display else None)
class IorCommand(ExecutableCommand): """Defines a object for executing an IOR command. Example: >>> # Typical use inside of a DAOS avocado test method. >>> ior_cmd = IorCommand() >>> ior_cmd.get_params(self) >>> ior_cmd.set_daos_params(self.server_group, self.pool) >>> mpirun = Mpirun() >>> env = self.ior_cmd.get_default_env(self.tmp, self.client_log) >>> processes = len(self.hostlist_clients) >>> mpirun.setup_command(env, self.hostfile_clients, processes) >>> mpirun.run() """ def __init__(self): """Create an IorCommand object.""" super(IorCommand, self).__init__("/run/ior/*", "ior") # Flags self.flags = FormattedParameter("{}") # Optional arguments # -a=POSIX API for I/O [POSIX|DUMMY|MPIIO|MMAP|DAOS|DFS] # -b=1048576 blockSize -- contiguous bytes to write per task # -d=0 interTestDelay -- delay between reps in seconds # -f=STRING scriptFile -- test script name # -G=0 setTimeStampSignature -- time stamp signature # -i=1 repetitions -- number of repetitions of test # -j=0 outlierThreshold -- warn on outlier N sec from mean # -J=1 setAlignment -- HDF5 alignment in bytes # -l=STRING datapacket type-- type of packet created # -M=STRING memoryPerNode -- hog memory on the node # -N=0 numTasks -- num of participating tasks in the test # -o=testFile testFile -- full name for test # -O=STRING string of IOR directives # -Q=1 taskPerNodeOffset for read tests # -s=1 segmentCount -- number of segments # -t=262144 transferSize -- size of transfer in bytes # -T=0 maxTimeDuration -- max time in minutes executing # repeated test; it aborts only between iterations # and not within a test! self.api = FormattedParameter("-a {}", "DAOS") self.block_size = FormattedParameter("-b {}") self.test_delay = FormattedParameter("-d {}") self.script = FormattedParameter("-f {}") self.signatute = FormattedParameter("-G {}") self.repetitions = FormattedParameter("-i {}") self.outlier_threshold = FormattedParameter("-j {}") self.alignment = FormattedParameter("-J {}") self.data_packet_type = FormattedParameter("-l {}") self.memory_per_node = FormattedParameter("-M {}") self.num_tasks = FormattedParameter("-N {}") self.test_file = FormattedParameter("-o {}") self.directives = FormattedParameter("-O {}") self.task_offset = FormattedParameter("-Q {}") self.segment_count = FormattedParameter("-s {}") self.transfer_size = FormattedParameter("-t {}") self.max_duration = FormattedParameter("-T {}") # Module DAOS # Required arguments # --daos.pool=STRING pool uuid # --daos.svcl=STRING pool SVCL # --daos.cont=STRING container uuid # Flags # --daos.destroy Destroy Container # Optional arguments # --daos.group=STRING server group # --daos.chunk_size=1048576 chunk size # --daos.oclass=STRING object class self.daos_pool = FormattedParameter("--daos.pool {}") self.daos_svcl = FormattedParameter("--daos.svcl {}") self.daos_cont = FormattedParameter("--daos.cont {}") self.daos_destroy = FormattedParameter("--daos.destroy", True) self.daos_group = FormattedParameter("--daos.group {}") self.daos_chunk = FormattedParameter("--daos.chunk_size {}", 1048576) self.daos_oclass = FormattedParameter("--daos.oclass {}", "SX") def get_param_names(self): """Get a sorted list of the defined IorCommand parameters.""" # Sort the IOR parameter names to generate consistent ior commands all_param_names = super(IorCommand, self).get_param_names() # List all of the common ior params first followed by any daos-specific # params (except when using MPIIO). param_names = [name for name in all_param_names if "daos" not in name] if self.api.value not in ["MPIIO", "POSIX"]: param_names.extend( [name for name in all_param_names if "daos" in name]) return param_names def set_daos_params(self, group, pool, cont_uuid=None, display=True): """Set the IOR parameters for the DAOS group, pool, and container uuid. Args: group (str): DAOS server group name pool (TestPool): DAOS test pool object cont_uuid (str, optional): the container uuid. If not specified one is generated. Defaults to None. display (bool, optional): print updated params. Defaults to True. """ self.set_daos_pool_params(pool, display) self.daos_group.update(group, "daos_group" if display else None) self.daos_cont.update(cont_uuid if cont_uuid else uuid.uuid4(), "daos_cont" if display else None) def set_daos_pool_params(self, pool, display=True): """Set the IOR parameters that are based on a DAOS pool. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ self.daos_pool.update(pool.pool.get_uuid_str(), "daos_pool" if display else None) self.set_daos_svcl_param(pool, display) def set_daos_svcl_param(self, pool, display=True): """Set the IOR daos_svcl param from the ranks of a DAOS pool object. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ svcl = ":".join([ str(item) for item in [ int(pool.pool.svc.rl_ranks[index]) for index in range(pool.pool.svc.rl_nr) ] ]) self.daos_svcl.update(svcl, "daos_svcl" if display else None) def get_aggregate_total(self, processes): """Get the total bytes expected to be written by ior. Args: processes (int): number of processes running the ior command Returns: int: total number of bytes written Raises: CommandFailure: if there is an error obtaining the aggregate total """ power = {"k": 1, "m": 2, "g": 3, "t": 4} total = processes for name in ("block_size", "segment_count"): item = getattr(self, name).value if item: sub_item = re.split(r"([^\d])", str(item)) if sub_item > 0: total *= int(sub_item[0]) if len(sub_item) > 1: key = sub_item[1].lower() if key in power: total *= 1024**power[key] else: raise CommandFailure( "Error obtaining the IOR aggregate total from " "the {} - bad key: value: {}, split: {}, " "key: {}".format(name, item, sub_item, key)) else: raise CommandFailure( "Error obtaining the IOR aggregate total from the {}: " "value: {}, split: {}".format(name, item, sub_item)) # Account for any replicas try: # Extract the replica quantity from the object class string replica_qty = int(re.findall(r"\d+", self.daos_oclass.value)[0]) except (TypeError, IndexError): # If the daos object class is undefined (TypeError) or it does not # contain any numbers (IndexError) then there is only one replica replica_qty = 1 finally: total *= replica_qty return total def get_default_env(self, manager_cmd, attach_info, log_file=None): """Get the default enviroment settings for running IOR. Args: manager_cmd (str): job manager command attach_info (str): CART attach info path log_file (str, optional): log file. Defaults to None. Returns: EnvironmentVariables: a dictionary of environment names and values """ env = EnvironmentVariables() env["CRT_ATTACH_INFO_PATH"] = attach_info env["MPI_LIB"] = "\"\"" env["DAOS_SINGLETON_CLI"] = 1 env["FI_PSM2_DISCONNECT"] = 1 if log_file: env["D_LOG_FILE"] = log_file if "mpirun" in manager_cmd or "srun" in manager_cmd: env["DAOS_POOL"] = self.daos_pool.value env["DAOS_SVCL"] = self.daos_svcl.value env["FI_PSM2_DISCONNECT"] = 1 env["IOR_HINT__MPI__romio_daos_obj_class"] = self.daos_oclass.value return env @staticmethod def get_ior_metrics(cmdresult): """Parse the CmdResult (output of the test) and look for the ior stdout and get the read and write metrics. Args: cmdresult (CmdResult): output of job manager Returns: metrics (tuple) : list of write and read metrics from ior run """ ior_metric_summary = "Summary of all tests:" messages = cmdresult.stdout.splitlines() # Get the index whre the summary starts and add one to # get to the header. idx = messages.index(ior_metric_summary) # idx + 1 is header. # idx +2 and idx + 3 will give the write and read metrics. write_metrics = (" ".join(messages[idx + 2].split())).split() read_metrics = (" ".join(messages[idx + 3].split())).split() return (write_metrics, read_metrics) @staticmethod def log_metrics(logger, message, metrics): """Log the ior metrics Args: logger (log): logger object handle message (str) : Message to print before logging metrics metric (lst) : IOR write and read metrics """ logger.info("\n") logger.info(message) for m in metrics: logger.info(m) logger.info("\n")
class DmgCommand(YamlCommand): """Defines a object representing a dmg command.""" METHOD_REGEX = { "run": r"(.*)", "network_scan": r"(?:|[-]+\s+(.*)\s+[-]+(?:\n|\n\r))" r"(?:.*\s+(fabric_iface|provider|pinned_numa_node):\s+" r"([a-z0-9+;_]+))", # Sample output of dmg pool list. # wolf-3:10001: connected # Pool UUID Svc Replicas # --------- ------------ # b4a27b5b-688a-4d1e-8c38-363e32eb4f29 1,2,3 # Between the first and the second group, use " +"; i.e., one or more # whitespaces. If we use "\s+", it'll pick up the second divider as # UUID since it's made up of hyphens and \s includes new line. "pool_list": r"(?:([0-9a-fA-F-]+) +([0-9,]+))" } def __init__(self, path, yaml_cfg=None): """Create a dmg Command object. Args: path (str): path to the dmg command yaml_cfg (DmgYamlParameters, optional): dmg config file settings. Defaults to None, in which case settings must be supplied as command-line paramters. """ super(DmgCommand, self).__init__("/run/dmg/*", "dmg", path, yaml_cfg) # If specified use the configuration file from the YamlParameters object default_yaml_file = None if isinstance(self.yaml, YamlParameters): default_yaml_file = self.yaml.filename self._hostlist = FormattedParameter("-l {}") self.hostfile = FormattedParameter("-f {}") self.configpath = FormattedParameter("-o {}", default_yaml_file) self.insecure = FormattedParameter("-i", False) self.debug = FormattedParameter("-d", False) self.json = FormattedParameter("-j", False) @property def hostlist(self): """Get the hostlist that was set. Returns a string list. """ if self.yaml: return self.yaml.hostlist.value else: return self._hostlist.value.split(",") @hostlist.setter def hostlist(self, hostlist): """Set the hostlist to be used for dmg invocation. Args: hostlist (string list): list of host addresses """ if self.yaml: if not isinstance(hostlist, list): hostlist = hostlist.split(",") self.yaml.hostlist.update(hostlist, "dmg.yaml.hostlist") else: if isinstance(hostlist, list): hostlist = ",".join(hostlist) self._hostlist.update(hostlist, "dmg._hostlist") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg sub command object based upon the sub-command.""" if self.sub_command.value == "network": self.sub_command_class = self.NetworkSubCommand() elif self.sub_command.value == "pool": self.sub_command_class = self.PoolSubCommand() elif self.sub_command.value == "storage": self.sub_command_class = self.StorageSubCommand() elif self.sub_command.value == "system": self.sub_command_class = self.SystemSubCommand() else: self.sub_command_class = None class NetworkSubCommand(CommandWithSubCommand): """Defines an object for the dmg network sub command.""" def __init__(self): """Create a dmg network subcommand object.""" super(DmgCommand.NetworkSubCommand, self).__init__("/run/dmg/network/*", "network") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg network sub command object.""" if self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() else: self.sub_command_class = None class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg network scan command.""" def __init__(self): """Create a dmg network scan command object.""" super(DmgCommand.NetworkSubCommand.ScanSubCommand, self).__init__("/run/dmg/network/scan/*", "scan") self.provider = FormattedParameter("-p {}", None) self.all = FormattedParameter("-a", False) class PoolSubCommand(CommandWithSubCommand): """Defines an object for the dmg pool sub command.""" def __init__(self): """Create a dmg pool subcommand object.""" super(DmgCommand.PoolSubCommand, self).__init__("/run/dmg/pool/*", "pool") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg pool sub command object.""" if self.sub_command.value == "create": self.sub_command_class = self.CreateSubCommand() elif self.sub_command.value == "delete-acl": self.sub_command_class = self.DeleteAclSubCommand() elif self.sub_command.value == "destroy": self.sub_command_class = self.DestroySubCommand() elif self.sub_command.value == "get-acl": self.sub_command_class = self.GetAclSubCommand() elif self.sub_command.value == "list": self.sub_command_class = self.ListSubCommand() elif self.sub_command.value == "overwrite-acl": self.sub_command_class = self.OverwriteAclSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "set-prop": self.sub_command_class = self.SetPropSubCommand() elif self.sub_command.value == "update-acl": self.sub_command_class = self.UpdateAclSubCommand() else: self.sub_command_class = None class CreateSubCommand(CommandWithParameters): """Defines an object for the dmg pool create command.""" def __init__(self): """Create a dmg pool create command object.""" super(DmgCommand.PoolSubCommand.CreateSubCommand, self).__init__("/run/dmg/pool/create/*", "create") self.group = FormattedParameter("--group={}", None) self.user = FormattedParameter("--user={}", None) self.acl_file = FormattedParameter("--acl-file={}", None) self.scm_size = FormattedParameter("--scm-size={}", None) self.nvme_size = FormattedParameter("--nvme-size={}", None) self.ranks = FormattedParameter("--ranks={}", None) self.nsvc = FormattedParameter("--nsvc={}", None) self.sys = FormattedParameter("--sys={}", None) class DeleteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool delete-acl command.""" def __init__(self): """Create a dmg pool delete-acl command object.""" super(DmgCommand.PoolSubCommand.DeleteAclSubCommand, self).__init__("/run/dmg/pool/delete-acl/*", "delete-acl") self.pool = FormattedParameter("--pool={}", None) self.principal = FormattedParameter("-p {}", None) class DestroySubCommand(CommandWithParameters): """Defines an object for the dmg pool destroy command.""" def __init__(self): """Create a dmg pool destroy command object.""" super(DmgCommand.PoolSubCommand.DestroySubCommand, self).__init__("/run/dmg/pool/destroy/*", "destroy") self.pool = FormattedParameter("--pool={}", None) self.sys_name = FormattedParameter("--sys-name={}", None) self.force = FormattedParameter("--force", False) class GetAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool get-acl command.""" def __init__(self): """Create a dmg pool get-acl command object.""" super(DmgCommand.PoolSubCommand.GetAclSubCommand, self).__init__("/run/dmg/pool/get-acl/*", "get-acl") self.pool = FormattedParameter("--pool={}", None) class ListSubCommand(CommandWithParameters): """Defines an object for the dmg pool list command.""" def __init__(self): """Create a dmg pool list command object.""" super(DmgCommand.PoolSubCommand.ListSubCommand, self).__init__("/run/dmg/pool/list/*", "list") class OverwriteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool overwrite-acl command.""" def __init__(self): """Create a dmg pool overwrite-acl command object.""" super(DmgCommand.PoolSubCommand.OverwriteAclSubCommand, self).__init__("/run/dmg/pool/overwrite-acl/*", "overwrite-acl") self.pool = FormattedParameter("--pool={}", None) self.acl_file = FormattedParameter("-a {}", None) class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg pool query command.""" def __init__(self): """Create a dmg pool query command object.""" super(DmgCommand.PoolSubCommand.QuerySubCommand, self).__init__("/run/dmg/pool/query/*", "query") self.pool = FormattedParameter("--pool={}", None) class SetPropSubCommand(CommandWithParameters): """Defines an object for the dmg pool set-prop command.""" def __init__(self): """Create a dmg pool set-prop command object.""" super(DmgCommand.PoolSubCommand.SetPropSubCommand, self).__init__("/run/dmg/pool/set-prop/*", "set-prop") self.pool = FormattedParameter("--pool={}", None) self.name = FormattedParameter("--name={}", None) self.value = FormattedParameter("--value={}", None) class UpdateAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool update-acl command.""" def __init__(self): """Create a dmg pool update-acl command object.""" super(DmgCommand.PoolSubCommand.UpdateAclSubCommand, self).__init__("/run/dmg/pool/update-acl/*", "update-acl") self.pool = FormattedParameter("--pool={}", None) self.acl_file = FormattedParameter("-a {}", None) self.entry = FormattedParameter("-e {}", None) class StorageSubCommand(CommandWithSubCommand): """Defines an object for the dmg storage sub command.""" def __init__(self): """Create a dmg storage subcommand object.""" super(DmgCommand.StorageSubCommand, self).__init__("/run/dmg/storage/*", "storage") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg storage sub command object.""" if self.sub_command.value == "format": self.sub_command_class = self.FormatSubCommand() elif self.sub_command.value == "prepare": self.sub_command_class = self.PrepareSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() elif self.sub_command.value == "set": self.sub_command_class = self.SetSubCommand() else: self.sub_command_class = None class FormatSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage format command object.""" super(DmgCommand.StorageSubCommand.FormatSubCommand, self).__init__("/run/dmg/storage/format/*", "format") self.reformat = FormattedParameter("--reformat", False) class PrepareSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage prepare command object.""" super(DmgCommand.StorageSubCommand.PrepareSubCommand, self).__init__("/run/dmg/storage/prepare/*", "prepare") self.pci_whitelist = FormattedParameter("-w {}", None) self.hugepages = FormattedParameter("-p {}", None) self.target_user = FormattedParameter("-u {}", None) self.nvme_only = FormattedParameter("-n", False) self.scm_only = FormattedParameter("-s", False) self.reset = FormattedParameter("--reset", False) self.force = FormattedParameter("-f", False) class QuerySubCommand(CommandWithSubCommand): """Defines an object for the dmg query format command.""" def __init__(self): """Create a dmg storage query command object.""" super(DmgCommand.StorageSubCommand.QuerySubCommand, self).__init__("/run/dmg/storage/query/*", "query") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg pool sub command object.""" if self.sub_command.value == "blobstore-health": self.sub_command_class = self.BlobstoreHealthSubCommand() elif self.sub_command.value == "smd": self.sub_command_class = self.SmdSubCommand() else: self.sub_command_class = None class BlobstoreHealthSubCommand(CommandWithParameters): """Defines a dmg storage query blobstore-health object.""" def __init__(self): """Create a dmg storage query blobstore-health object.""" super( DmgCommand.StorageSubCommand.QuerySubCommand. BlobstoreHealthSubCommand, self).__init__( "/run/dmg/storage/query/blobstore-health/*", "blobstore-health") self.devuuid = FormattedParameter("-u {}", None) self.tgtid = FormattedParameter("-t {}", None) class SmdSubCommand(CommandWithParameters): """Defines a dmg storage query smd object.""" def __init__(self): """Create a dmg storage query smd object.""" super( DmgCommand.StorageSubCommand.QuerySubCommand. SmdSubCommand, self).__init__("/run/dmg/storage/query/smd/*", "smd") self.devices = FormattedParameter("-d", False) self.pools = FormattedParameter("-p", False) class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg storage scan command.""" def __init__(self): """Create a dmg storage scan command object.""" super(DmgCommand.StorageSubCommand.ScanSubCommand, self).__init__("/run/dmg/storage/scan/*", "scan") self.summary = FormattedParameter("-m", False) class SetSubCommand(CommandWithParameters): """Defines an object for the dmg storage set command.""" def __init__(self): """Create a dmg storage set command object.""" super(DmgCommand.StorageSubCommand.SetSubCommand, self).__init__("/run/dmg/storage/set/*", "set") self.nvme_faulty = FormattedParameter("nvme-faulty", False) class SystemSubCommand(CommandWithSubCommand): """Defines an object for the dmg system sub command.""" def __init__(self): """Create a dmg system subcommand object.""" super(DmgCommand.SystemSubCommand, self).__init__("/run/dmg/system/*", "system") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg system sub command object.""" if self.sub_command.value == "leader-query": self.sub_command_class = self.LeaderQuerySubCommand() elif self.sub_command.value == "list-pools": self.sub_command_class = self.ListPoolsSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "start": self.sub_command_class = self.StartSubCommand() elif self.sub_command.value == "stop": self.sub_command_class = self.StopSubCommand() else: self.sub_command_class = None class LeaderQuerySubCommand(CommandWithParameters): """Defines an object for the dmg system leader-query command.""" def __init__(self): """Create a dmg system leader-query command object.""" super(DmgCommand.SystemSubCommand.LeaderQuerySubCommand, self).__init__("/run/dmg/system/leader-query/*", "leader-query") class ListPoolsSubCommand(CommandWithParameters): """Defines an object for the dmg system list-pools command.""" def __init__(self): """Create a dmg system list-pools command object.""" super(DmgCommand.SystemSubCommand.ListPoolsSubCommand, self).__init__("/run/dmg/system/list-pools/*", "list-pools") class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg system query command.""" def __init__(self): """Create a dmg system query command object.""" super(DmgCommand.SystemSubCommand.QuerySubCommand, self).__init__("/run/dmg/system/query/*", "query") self.rank = FormattedParameter("--rank={}") self.verbose = FormattedParameter("--verbose", False) class StartSubCommand(CommandWithParameters): """Defines an object for the dmg system start command.""" def __init__(self): """Create a dmg system start command object.""" super(DmgCommand.SystemSubCommand.StartSubCommand, self).__init__("/run/dmg/system/start/*", "start") class StopSubCommand(CommandWithParameters): """Defines an object for the dmg system stop command.""" def __init__(self): """Create a dmg system stop command object.""" super(DmgCommand.SystemSubCommand.StopSubCommand, self).__init__("/run/dmg/system/stop/*", "stop") self.force = FormattedParameter("--force", False) def _get_result(self): """Get the result from running the configured dmg command. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg command fails. """ if self.yaml: self.create_yaml_file() result = None try: result = self.run() except CommandFailure as error: raise CommandFailure("<dmg> command failed: {}".format(error)) return result def network_scan(self, provider=None, all_devs=False): """Get the result of the dmg network scan command. Args: provider (str): name of network provider tied to the device all_devs (bool, optional): Show all devs info. Defaults to False. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage scan command fails. """ self.set_sub_command("network") self.sub_command_class.set_sub_command("scan") self.sub_command_class.sub_command_class.provider.value = provider self.sub_command_class.sub_command_class.all.value = all_devs return self._get_result() def storage_scan(self): """Get the result of the dmg storage scan command. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage scan command fails. """ self.set_sub_command("storage") self.sub_command_class.set_sub_command("scan") return self._get_result() def storage_format(self): """Get the result of the dmg storage format command. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage format command fails. """ self.set_sub_command("storage") self.sub_command_class.set_sub_command("format") return self._get_result() def storage_prepare(self, user=None, hugepages="4096", nvme=False, scm=False, reset=False, force=True): """Get the result of the dmg storage format command. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage prepare command fails. """ self.set_sub_command("storage") self.sub_command_class.set_sub_command("prepare") self.sub_command_class.sub_command_class.nvme_only.value = nvme self.sub_command_class.sub_command_class.scm_only.value = scm self.sub_command_class.sub_command_class.target_user.value = \ getuser() if user is None else user self.sub_command_class.sub_command_class.hugepages.value = hugepages self.sub_command_class.sub_command_class.reset.value = reset self.sub_command_class.sub_command_class.force.value = force return self._get_result() def pool_create(self, scm_size, uid=None, gid=None, nvme_size=None, target_list=None, svcn=None, group=None, acl_file=None): """Create a pool with the dmg command. The uid and gid method arguments can be specified as either an integer or a string. If an integer value is specified it will be converted into the corresponding user/group name string. Args: scm_size (int): SCM pool size to create. uid (object, optional): User ID with privileges. Defaults to None. gid (object, otional): Group ID with privileges. Defaults to None. nvme_size (str, optional): NVMe size. Defaults to None. target_list (list, optional): a list of storage server unique identifiers (ranks) for the DAOS pool svcn (str, optional): Number of pool service replicas. Defaults to None, in which case 1 is used by the dmg binary in default. group (str, optional): DAOS system group name in which to create the pool. Defaults to None, in which case "daos_server" is used by default. acl_file (str, optional): ACL file. Defaults to None. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg pool create command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("create") self.sub_command_class.sub_command_class.user.value = \ getpwuid(uid).pw_name if isinstance(uid, int) else uid self.sub_command_class.sub_command_class.group.value = \ getgrgid(gid).gr_name if isinstance(gid, int) else gid self.sub_command_class.sub_command_class.scm_size.value = scm_size self.sub_command_class.sub_command_class.nvme_size.value = nvme_size if target_list is not None: self.sub_command_class.sub_command_class.ranks.value = ",".join( [str(target) for target in target_list]) self.sub_command_class.sub_command_class.nsvc.value = svcn self.sub_command_class.sub_command_class.sys.value = group self.sub_command_class.sub_command_class.acl_file.value = acl_file return self._get_result() def pool_destroy(self, pool, force=True): """Destroy a pool with the dmg command. Args: pool (str): Pool UUID to destroy. force (bool, optional): Force removal of pool. Defaults to True. Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool destroy command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("destroy") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.force.value = force return self._get_result() def pool_get_acl(self, pool): """Get the ACL for a given pool. Args: pool (str): Pool for which to get the ACL. Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool get-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("get-acl") self.sub_command_class.sub_command_class.pool.value = pool return self._get_result() def pool_update_acl(self, pool, acl_file, entry): """Update the acl for a given pool. Args: pool (str): Pool for which to update the ACL. acl_file (str): ACL file to update entry (str): entry to be updated Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool update-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("update-acl") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.acl_file.value = acl_file self.sub_command_class.sub_command_class.entry.value = entry return self._get_result() def pool_overwrite_acl(self, pool, acl_file): """Overwrite the acl for a given pool. Args: pool (str): Pool for which to overwrite the ACL. acl_file (str): ACL file to update Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool overwrite-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("overwrite-acl") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.acl_file.value = acl_file return self._get_result() def pool_delete_acl(self, pool, principal): """Delete the acl for a given pool. Args: pool (str): Pool for which to delete the ACL. principal (str): principal to be deleted Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool delete-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("delete-acl") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.principal.value = principal return self._get_result() def pool_list(self): """List pools. Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool delete-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("list") return self._get_result() def pool_set_prop(self, pool, name, value): """Set property for a given Pool. Args: pool (str): Pool uuid for which property is supposed to be set. name (str): Property name to be set value (str): Property value to be set Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool set-prop command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("set-prop") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.name.value = name self.sub_command_class.sub_command_class.value.value = value return self._get_result()
class DmgCommand(CommandWithSubCommand): """Defines a object representing a dmg command.""" def __init__(self, path): """Create a dmg Command object. Args: path (str): path to the dmg command """ super(DmgCommand, self).__init__("/run/dmg/*", "dmg", path) self.hostlist = FormattedParameter("-l {}") self.hostfile = FormattedParameter("-f {}") self.configpath = FormattedParameter("-o {}") self.insecure = FormattedParameter("-i", True) self.debug = FormattedParameter("-d", False) self.json = FormattedParameter("-j", False) def set_hostlist(self, manager): """Set the dmg hostlist parameter with the daos server/agent info. Use the daos server/agent access points port and list of hosts to define the dmg --hostlist command line parameter. Args: manager (SubprocessManager): daos server/agent process manager """ self.hostlist.update(manager.get_config_value("access_points"), "dmg.hostlist") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg sub command object based upon the sub-command.""" if self.sub_command.value == "network": self.sub_command_class = self.NetworkSubCommand() elif self.sub_command.value == "pool": self.sub_command_class = self.PoolSubCommand() elif self.sub_command.value == "storage": self.sub_command_class = self.StorageSubCommand() elif self.sub_command.value == "system": self.sub_command_class = self.SystemSubCommand() else: self.sub_command_class = None class NetworkSubCommand(CommandWithSubCommand): """Defines an object for the dmg network sub command.""" def __init__(self): """Create a dmg network subcommand object.""" super(DmgCommand.NetworkSubCommand, self).__init__("/run/dmg/network/*", "network") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg network sub command object.""" if self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() else: self.sub_command_class = None class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg network scan command.""" def __init__(self): """Create a dmg network scan command object.""" super(DmgCommand.NetworkSubCommand.ScanSubCommand, self).__init__("/run/dmg/network/scan/*", "scan") self.provider = FormattedParameter("-p {}", None) self.all = FormattedParameter("-a", False) class PoolSubCommand(CommandWithSubCommand): """Defines an object for the dmg pool sub command.""" def __init__(self): """Create a dmg pool subcommand object.""" super(DmgCommand.PoolSubCommand, self).__init__("/run/dmg/pool/*", "pool") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg pool sub command object.""" if self.sub_command.value == "create": self.sub_command_class = self.CreateSubCommand() elif self.sub_command.value == "delete-acl": self.sub_command_class = self.DeleteAclSubCommand() elif self.sub_command.value == "destroy": self.sub_command_class = self.DestroySubCommand() elif self.sub_command.value == "get-acl": self.sub_command_class = self.GetAclSubCommand() elif self.sub_command.value == "list": self.sub_command_class = self.ListSubCommand() elif self.sub_command.value == "overwrite-acl": self.sub_command_class = self.OverwriteAclSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "set-prop": self.sub_command_class = self.SetPropSubCommand() elif self.sub_command.value == "update-acl": self.sub_command_class = self.UpdateAclSubCommand() else: self.sub_command_class = None class CreateSubCommand(CommandWithParameters): """Defines an object for the dmg pool create command.""" def __init__(self): """Create a dmg pool create command object.""" super(DmgCommand.PoolSubCommand.CreateSubCommand, self).__init__("/run/dmg/pool/create/*", "create") self.group = FormattedParameter("--group={}", None) self.user = FormattedParameter("--user={}", None) self.acl_file = FormattedParameter("--acl-file={}", None) self.scm_size = FormattedParameter("--scm-size={}", None) self.nvme_size = FormattedParameter("--nvme-size={}", None) self.ranks = FormattedParameter("--ranks={}", None) self.nsvc = FormattedParameter("--nsvc={}", None) self.sys = FormattedParameter("--sys={}", None) class DeleteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool delete-acl command.""" def __init__(self): """Create a dmg pool delete-acl command object.""" super(DmgCommand.PoolSubCommand.DeleteAclSubCommand, self).__init__("/run/dmg/pool/delete-acl/*", "delete-acl") self.pool = FormattedParameter("--pool={}", None) self.principal = FormattedParameter("-p {}", None) class DestroySubCommand(CommandWithParameters): """Defines an object for the dmg pool destroy command.""" def __init__(self): """Create a dmg pool destroy command object.""" super(DmgCommand.PoolSubCommand.DestroySubCommand, self).__init__("/run/dmg/pool/destroy/*", "destroy") self.pool = FormattedParameter("--pool={}", None) self.sys_name = FormattedParameter("--sys-name={}", None) self.force = FormattedParameter("--force", False) class GetAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool get-acl command.""" def __init__(self): """Create a dmg pool get-acl command object.""" super(DmgCommand.PoolSubCommand.GetAclSubCommand, self).__init__("/run/dmg/pool/get-acl/*", "get-acl") self.pool = FormattedParameter("--pool={}", None) class ListSubCommand(CommandWithParameters): """Defines an object for the dmg pool list command.""" def __init__(self): """Create a dmg pool list command object.""" super(DmgCommand.PoolSubCommand.ListSubCommand, self).__init__("/run/dmg/pool/list/*", "list") class OverwriteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool overwrite-acl command.""" def __init__(self): """Create a dmg pool overwrite-acl command object.""" super(DmgCommand.PoolSubCommand.OverwriteAclSubCommand, self).__init__("/run/dmg/pool/overwrite-acl/*", "overwrite-acl") self.pool = FormattedParameter("--pool={}", None) self.acl_file = FormattedParameter("-a {}", None) class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg pool query command.""" def __init__(self): """Create a dmg pool query command object.""" super(DmgCommand.PoolSubCommand.QuerySubCommand, self).__init__("/run/dmg/pool/query/*", "query") self.pool = FormattedParameter("--pool={}", None) class SetPropSubCommand(CommandWithParameters): """Defines an object for the dmg pool set-prop command.""" def __init__(self): """Create a dmg pool set-prop command object.""" super(DmgCommand.PoolSubCommand.SetPropSubCommand, self).__init__("/run/dmg/pool/set-prop/*", "set-prop") self.pool = FormattedParameter("--pool={}", None) self.name = FormattedParameter("--name={}", None) self.value = FormattedParameter("--value={}", None) class UpdateAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool update-acl command.""" def __init__(self): """Create a dmg pool update-acl command object.""" super(DmgCommand.PoolSubCommand.UpdateAclSubCommand, self).__init__("/run/dmg/pool/update-acl/*", "update-acl") self.pool = FormattedParameter("--pool={}", None) self.acl_file = FormattedParameter("-a {}", None) self.entry = FormattedParameter("-e {}", None) class StorageSubCommand(CommandWithSubCommand): """Defines an object for the dmg storage sub command.""" def __init__(self): """Create a dmg storage subcommand object.""" super(DmgCommand.StorageSubCommand, self).__init__("/run/dmg/storage/*", "storage") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg storage sub command object.""" if self.sub_command.value == "format": self.sub_command_class = self.FormatSubCommand() elif self.sub_command.value == "prepare": self.sub_command_class = self.PrepareSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() elif self.sub_command.value == "set": self.sub_command_class = self.SetSubCommand() else: self.sub_command_class = None class FormatSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage format command object.""" super(DmgCommand.StorageSubCommand.FormatSubCommand, self).__init__("/run/dmg/storage/format/*", "format") self.reformat = FormattedParameter("--reformat", False) class PrepareSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage prepare command object.""" super(DmgCommand.StorageSubCommand.PrepareSubCommand, self).__init__("/run/dmg/storage/prepare/*", "prepare") self.pci_whitelist = FormattedParameter("-w {}", None) self.hugepages = FormattedParameter("-p {}", None) self.target_user = FormattedParameter("-u {}", None) self.nvme_only = FormattedParameter("-n", False) self.scm_only = FormattedParameter("-s", False) self.reset = FormattedParameter("--reset", False) self.force = FormattedParameter("-f", False) class QuerySubCommand(CommandWithSubCommand): """Defines an object for the dmg query format command.""" def __init__(self): """Create a dmg storage query command object.""" super(DmgCommand.StorageSubCommand.QuerySubCommand, self).__init__("/run/dmg/storage/query/*", "query") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg pool sub command object.""" if self.sub_command.value == "blobstore-health": self.sub_command_class = self.BlobstoreHealthSubCommand() elif self.sub_command.value == "smd": self.sub_command_class = self.SmdSubCommand() else: self.sub_command_class = None class BlobstoreHealthSubCommand(CommandWithParameters): """Defines a dmg storage query blobstore-health object.""" def __init__(self): """Create a dmg storage query blobstore-health object.""" super( DmgCommand.StorageSubCommand.QuerySubCommand. BlobstoreHealthSubCommand, self).__init__( "/run/dmg/storage/query/blobstore-health/*", "blobstore-health") self.devuuid = FormattedParameter("-u {}", None) self.tgtid = FormattedParameter("-t {}", None) class SmdSubCommand(CommandWithParameters): """Defines a dmg storage query smd object.""" def __init__(self): """Create a dmg storage query smd object.""" super( DmgCommand.StorageSubCommand.QuerySubCommand. SmdSubCommand, self).__init__("/run/dmg/storage/query/smd/*", "smd") self.devices = FormattedParameter("-d", False) self.pools = FormattedParameter("-p", False) class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg storage scan command.""" def __init__(self): """Create a dmg storage scan command object.""" super(DmgCommand.StorageSubCommand.ScanSubCommand, self).__init__("/run/dmg/storage/scan/*", "scan") self.summary = FormattedParameter("-m", False) class SetSubCommand(CommandWithParameters): """Defines an object for the dmg storage set command.""" def __init__(self): """Create a dmg storage set command object.""" super(DmgCommand.StorageSubCommand.SetSubCommand, self).__init__("/run/dmg/storage/set/*", "set") self.nvme_faulty = FormattedParameter("nvme-faulty", False) class SystemSubCommand(CommandWithSubCommand): """Defines an object for the dmg system sub command.""" def __init__(self): """Create a dmg system subcommand object.""" super(DmgCommand.SystemSubCommand, self).__init__("/run/dmg/system/*", "system") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg system sub command object.""" if self.sub_command.value == "leader-query": self.sub_command_class = self.LeaderQuerySubCommand() elif self.sub_command.value == "list-pools": self.sub_command_class = self.ListPoolsSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "start": self.sub_command_class = self.StartSubCommand() elif self.sub_command.value == "stop": self.sub_command_class = self.StopSubCommand() else: self.sub_command_class = None class LeaderQuerySubCommand(CommandWithParameters): """Defines an object for the dmg system leader-query command.""" def __init__(self): """Create a dmg system leader-query command object.""" super(DmgCommand.SystemSubCommand.LeaderQuerySubCommand, self).__init__("/run/dmg/system/leader-query/*", "leader-query") class ListPoolsSubCommand(CommandWithParameters): """Defines an object for the dmg system list-pools command.""" def __init__(self): """Create a dmg system list-pools command object.""" super(DmgCommand.SystemSubCommand.ListPoolsSubCommand, self).__init__("/run/dmg/system/list-pools/*", "list-pools") class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg system query command.""" def __init__(self): """Create a dmg system query command object.""" super(DmgCommand.SystemSubCommand.QuerySubCommand, self).__init__("/run/dmg/system/query/*", "query") self.rank = FormattedParameter("--rank={}") self.verbose = FormattedParameter("--verbose", False) class StartSubCommand(CommandWithParameters): """Defines an object for the dmg system start command.""" def __init__(self): """Create a dmg system start command object.""" super(DmgCommand.SystemSubCommand.StartSubCommand, self).__init__("/run/dmg/system/start/*", "start") class StopSubCommand(CommandWithParameters): """Defines an object for the dmg system stop command.""" def __init__(self): """Create a dmg system stop command object.""" super(DmgCommand.SystemSubCommand.StopSubCommand, self).__init__("/run/dmg/system/stop/*", "stop") self.force = FormattedParameter("--force", False) def _get_result(self): """Get the result from running the configured dmg command. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg command fails. """ result = None try: result = self.run() except CommandFailure as error: raise CommandFailure("<dmg> command failed: {}".format(error)) return result def storage_scan(self): """Get the result of the dmg storage scan command. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage scan command fails. """ self.set_sub_command("storage") self.sub_command_class.set_sub_command("scan") return self._get_result() def storage_format(self): """Get the result of the dmg storage format command. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage format command fails. """ self.set_sub_command("storage") self.sub_command_class.set_sub_command("format") return self._get_result() def storage_prepare(self, user=None, hugepages="4096", nvme=False, scm=False, reset=False, force=True): """Get the result of the dmg storage format command. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage prepare command fails. """ self.set_sub_command("storage") self.sub_command_class.set_sub_command("prepare") self.sub_command_class.sub_command_class.nvme_only.value = nvme self.sub_command_class.sub_command_class.scm_only.value = scm self.sub_command_class.sub_command_class.target_user.value = \ getuser() if user is None else user self.sub_command_class.sub_command_class.hugepages.value = hugepages self.sub_command_class.sub_command_class.reset.value = reset self.sub_command_class.sub_command_class.force.value = force return self._get_result() def pool_create(self, scm_size, uid=None, gid=None, nvme_size=None, target_list=None, svcn=None, group=None, acl_file=None): """Create a pool with the dmg command. The uid and gid method arguments can be specified as either an integer or a string. If an integer value is specified it will be converted into the corresponding user/group name string. Args: scm_size (int): SCM pool size to create. uid (object, optional): User ID with privileges. Defaults to None. gid (object, otional): Group ID with privileges. Defaults to None. nvme_size (str, optional): NVMe size. Defaults to None. target_list (list, optional): a list of storage server unique identifiers (ranks) for the DAOS pool svcn (str, optional): Number of pool service replicas. Defaults to None, in which case 1 is used by the dmg binary in default. group (str, optional): DAOS system group name in which to create the pool. Defaults to None, in which case "daos_server" is used by default. acl_file (str, optional): ACL file. Defaults to None. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg pool create command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("create") self.sub_command_class.sub_command_class.user.value = \ getpwuid(uid).pw_name if isinstance(uid, int) else uid self.sub_command_class.sub_command_class.group.value = \ getgrgid(gid).gr_name if isinstance(gid, int) else gid self.sub_command_class.sub_command_class.scm_size.value = scm_size self.sub_command_class.sub_command_class.nvme_size.value = nvme_size if target_list is not None: self.sub_command_class.sub_command_class.ranks.value = ",".join( [str(target) for target in target_list]) self.sub_command_class.sub_command_class.nsvc.value = svcn self.sub_command_class.sub_command_class.sys.value = group self.sub_command_class.sub_command_class.acl_file.value = acl_file return self._get_result() def pool_destroy(self, pool, force=True): """Destroy a pool with the dmg command. Args: pool (str): Pool UUID to destroy. force (bool, optional): Force removal of pool. Defaults to True. Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool destroy command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("destroy") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.force.value = force return self._get_result() def pool_get_acl(self, pool): """Get the ACL for a given pool. Args: pool (str): Pool for which to get the ACL. Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool get-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("get-acl") self.sub_command_class.sub_command_class.pool.value = pool return self._get_result() def pool_update_acl(self, pool, acl_file, entry): """Update the acl for a given pool. Args: pool (str): Pool for which to update the ACL. acl_file (str): ACL file to update entry (str): entry to be updated Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool update-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("update-acl") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.acl_file.value = acl_file self.sub_command_class.sub_command_class.entry.value = entry return self._get_result() def pool_overwrite_acl(self, pool, acl_file): """Overwrite the acl for a given pool. Args: pool (str): Pool for which to overwrite the ACL. acl_file (str): ACL file to update Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool overwrite-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("overwrite-acl") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.acl_file.value = acl_file return self._get_result() def pool_delete_acl(self, pool, principal): """Delete the acl for a given pool. Args: pool (str): Pool for which to delete the ACL. principal (str): principal to be deleted Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool delete-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("delete-acl") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.principal.value = principal return self._get_result()
class MdtestCommand(ExecutableCommand): """Defines a object representing a mdtest command.""" def __init__(self): """Create an MdtestCommand object.""" super(MdtestCommand, self).__init__("/run/mdtest/*", "mdtest") self.flags = FormattedParameter("{}") # mdtest flags # Optional arguments # -a=STRING API for I/O [POSIX|DUMMY] # -b=1 branching factor of hierarchical dir structure # -d=./out the directory in which the tests will run # -B=0 no barriers between phases # -e=0 bytes to read from each file # -f=1 first number of tasks on which test will run # -i=1 number of iterations the test will run # -I=0 number of items per directory in tree # -l=0 last number of tasks on which test will run # -n=0 every process will creat/stat/read/remove num # of directories and files # -N=0 stride num between neighbor tasks for file/dir # operation (local=0) # -p=0 pre-iteration delay (in seconds) # --random-seed=0 random seed for -R # -s=1 stride between number of tasks for each test # -V=0 verbosity value # -w=0 bytes to write each file after it is created # -W=0 number in seconds; stonewall timer, write as # many seconds and ensure all processes did the # same number of operations (currently only # stops during create phase) # -x=STRING StoneWallingStatusFile; contains the number # of iterations of the creation phase, can be # used to split phases across runs # -z=0 depth of hierarchical directory structure self.api = FormattedParameter("-a {}") self.branching_factor = FormattedParameter("-b {}") self.test_dir = FormattedParameter("-d {}") self.barriers = FormattedParameter("-B {}") self.read_bytes = FormattedParameter("-e {}") self.first_num_tasks = FormattedParameter("-f {}") self.iteration = FormattedParameter("-i {}") self.items = FormattedParameter("-I {}") self.last_num_tasks = FormattedParameter("-l {}") self.num_of_files_dirs = FormattedParameter("-n {}") self.pre_iter = FormattedParameter("-p {}") self.random_seed = FormattedParameter("--random-seed {}") self.stride = FormattedParameter("-s {}") self.verbosity_value = FormattedParameter("-V {}") self.write_bytes = FormattedParameter("-w {}") self.stonewall_timer = FormattedParameter("-W {}") self.stonewall_statusfile = FormattedParameter("-x {}") self.depth = FormattedParameter("-z {}") # Module DAOS (Not intended to be used as of now, hence all # arguments for DAOS module are commented) # Required arguments # --daos.pool=STRING pool uuid # --daos.svcl=STRING pool SVCL # --daos.cont=STRING container uuid # Flags # --daos.destroy Destroy Container # Optional arguments # --daos.group=STRING server group # --daos.chunk_size=1048576 chunk size # --daos.oclass=STRING object class # self.daos_pool_uuid = FormattedParameter("--daos.pool {}") # self.daos_svcl = FormattedParameter("--daos.svcl {}") # self.daos_cont = FormattedParameter("--daos.cont {}") # self.daos_group = FormattedParameter("--daos.group {}") # self.daos_chunk_size = FormattedParameter(" --daos.chunk_size {}") # self.daos_oclass = FormattedParameter("--daos.oclass {}") # self.daos_destroy = FormattedParameter("--daos.destroy", True) # Module DFS # Required arguments # --dfs.pool=STRING DAOS pool uuid # --dfs.svcl=STRING DAOS pool SVCL # --dfs.cont=STRING DFS container uuid # Flags # --dfs.destroy Destroy DFS Container # Optional arguments # --dfs.group=STRING DAOS server group self.dfs_pool_uuid = FormattedParameter("--dfs.pool {}") self.dfs_svcl = FormattedParameter("--dfs.svcl {}") self.dfs_cont = FormattedParameter("--dfs.cont {}") self.dfs_group = FormattedParameter("--dfs.group {}") self.dfs_destroy = FormattedParameter("--dfs.destroy", True) def get_param_names(self): """Get a sorted list of the defined MdtestCommand parameters.""" # Sort the Mdtest parameter names to generate consistent ior commands all_param_names = super(MdtestCommand, self).get_param_names() # List all of the common ior params first followed by any dfs-specific # params (except when using POSIX). param_names = [name for name in all_param_names if "dfs" not in name] if self.api.value != "POSIX": param_names.extend( [name for name in all_param_names if "dfs" in name]) return param_names def set_daos_params(self, group, pool, cont_uuid=None, display=True): """Set the Mdtest params for the DAOS group, pool, and container uuid. Args: group (str): DAOS server group name pool (TestPool): DAOS test pool object cont_uuid (str, optional): the container uuid. If not specified one is generated. Defaults to None. display (bool, optional): print updated params. Defaults to True. """ self.set_daos_pool_params(pool, display) self.dfs_group.update(group, "dfs_group" if display else None) self.dfs_cont.update( cont_uuid if cont_uuid else uuid.uuid4(), "dfs_cont" if display else None) def set_daos_pool_params(self, pool, display=True): """Set the Mdtest parameters that are based on a DAOS pool. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ self.dfs_pool_uuid.update( pool.pool.get_uuid_str(), "dfs_pool" if display else None) self.set_daos_svcl_param(pool, display) def set_daos_svcl_param(self, pool, display=True): """Set the Mdtest daos_svcl param from the ranks of a DAOS pool object. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ svcl = ":".join( [str(item) for item in [ int(pool.pool.svc.rl_ranks[index]) for index in range(pool.pool.svc.rl_nr)]]) self.dfs_svcl.update(svcl, "dfs_svcl" if display else None) def get_default_env(self, manager_cmd, attach_info, log_file=None): """Get the default enviroment settings for running mdtest. Args: manager_cmd (str): job manager command attach_info (str): CART attach info path log_file (str, optional): log file. Defaults to None. Returns: EnvironmentVariables: a dictionary of environment names and values """ env = EnvironmentVariables() env["CRT_ATTACH_INFO_PATH"] = attach_info env["MPI_LIB"] = "\"\"" env["DAOS_SINGLETON_CLI"] = 1 env["FI_PSM2_DISCONNECT"] = 1 if log_file: env["D_LOG_FILE"] = log_file if "mpirun" in manager_cmd or "srun" in manager_cmd: env["DAOS_POOL"] = self.dfs_pool_uuid.value env["DAOS_SVCL"] = self.dfs_svcl.value env["FI_PSM2_DISCONNECT"] = 1 return env
class IorCommand(CommandWithParameters): """Defines a object for executing an IOR command. Example: >>> # Typical use inside of a DAOS avocado test method. >>> ior_cmd = IorCommand() >>> ior_cmd.get_params(self) >>> ior_cmd.set_daos_params(self.server_group, self.pool) >>> ior_cmd.run( self.basepath, len(self.hostlist_clients), self.hostfile_clients) """ def __init__(self): """Create an IorCommand object.""" super(IorCommand, self).__init__("ior") # Flags self.flags = FormattedParameter("{}") # Optional arguments # -a=POSIX API for I/O [POSIX|DUMMY|MPIIO|MMAP|DAOS|DFS] # -b=1048576 blockSize -- contiguous bytes to write per task # -d=0 interTestDelay -- delay between reps in seconds # -f=STRING scriptFile -- test script name # -G=0 setTimeStampSignature -- time stamp signature # -i=1 repetitions -- number of repetitions of test # -j=0 outlierThreshold -- warn on outlier N sec from mean # -J=1 setAlignment -- HDF5 alignment in bytes # -l=STRING datapacket type-- type of packet created # -M=STRING memoryPerNode -- hog memory on the node # -N=0 numTasks -- num of participating tasks in the test # -o=testFile testFile -- full name for test # -O=STRING string of IOR directives # -Q=1 taskPerNodeOffset for read tests # -s=1 segmentCount -- number of segments # -t=262144 transferSize -- size of transfer in bytes # -T=0 maxTimeDuration -- max time in minutes executing # repeated test; it aborts only between iterations # and not within a test! self.api = FormattedParameter("-a {}", "DAOS") self.block_size = FormattedParameter("-b {}") self.test_delay = FormattedParameter("-d {}") self.script = FormattedParameter("-f {}") self.signatute = FormattedParameter("-G {}") self.repetitions = FormattedParameter("-i {}") self.outlier_threshold = FormattedParameter("-j {}") self.alignment = FormattedParameter("-J {}") self.data_packet_type = FormattedParameter("-l {}") self.memory_per_node = FormattedParameter("-M {}") self.num_tasks = FormattedParameter("-N {}") self.test_file = FormattedParameter("-o {}") self.directives = FormattedParameter("-O {}") self.task_offset = FormattedParameter("-Q {}") self.segment_count = FormattedParameter("-s {}") self.transfer_size = FormattedParameter("-t {}") self.max_duration = FormattedParameter("-T {}") # Module DAOS # Required arguments # --daos.pool=STRING pool uuid # --daos.svcl=STRING pool SVCL # --daos.cont=STRING container uuid # Flags # --daos.destroy Destroy Container # Optional arguments # --daos.group=STRING server group # --daos.chunk_size=1048576 chunk size # --daos.oclass=STRING object class self.daos_pool = FormattedParameter("--daos.pool {}") self.daos_svcl = FormattedParameter("--daos.svcl {}") self.daos_cont = FormattedParameter("--daos.cont {}") self.daos_destroy = FormattedParameter("--daos.destroy", True) self.daos_group = FormattedParameter("--daos.group {}") self.daos_chunk = FormattedParameter("--daos.chunk_size {}", 1048576) self.daos_oclass = FormattedParameter("--daos.oclass {}") def get_param_names(self): """Get a sorted list of the defined IorCommand parameters.""" # Sort the IOR parameter names to generate consistent ior commands all_param_names = super(IorCommand, self).get_param_names() # List all of the common ior params first followed by any daos-specific # params (except when using MPIIO). param_names = [name for name in all_param_names if "daos" not in name] if self.api.value != "MPIIO": param_names.extend( [name for name in all_param_names if "daos" in name]) return param_names def get_params(self, test, path="/run/ior/*"): """Get values for all of the ior command params using a yaml file. Sets each BasicParameter object's value to the yaml key that matches the assigned name of the BasicParameter object in this class. For example, the self.block_size.value will be set to the value in the yaml file with the key 'block_size'. Args: test (Test): avocado Test object path (str, optional): yaml namespace. Defaults to "/run/ior/*". """ super(IorCommand, self).get_params(test, path) def set_daos_params(self, group, pool, cont_uuid=None, display=True, mpiio_oclass=None): """Set the IOR parameters for the DAOS group, pool, and container uuid. Args: group (str): DAOS server group name pool (DaosPool): DAOS pool API object cont_uuid (str, optional): the container uuid. If not specified one is generated. Defaults to None. display (bool, optional): print updated params. Defaults to True. """ self.set_daos_pool_params(pool, display) self.daos_group.update(group, "daos_group" if display else None) self.daos_cont.update(cont_uuid if cont_uuid else uuid.uuid4(), "daos_cont" if display else None) # assigning obj class as SX in None else # the desired one if mpiio_oclass is None: self.mpiio_oclass = 214 else: self.mpiio_oclass = mpiio_oclass def set_daos_pool_params(self, pool, display=True): """Set the IOR parameters that are based on a DAOS pool. Args: pool (DaosPool): DAOS pool API object display (bool, optional): print updated params. Defaults to True. """ #self.daos_pool.value = pool.uuid self.daos_pool.update(pool.pool.get_uuid_str(), "daos_pool" if display else None) self.set_daos_svcl_param(pool, display) def set_daos_svcl_param(self, pool, display=True): """Set the IOR daos_svcl param from the ranks of a DAOS pool object. Args: pool (DaosPool): DAOS pool API object display (bool, optional): print updated params. Defaults to True. """ svcl = ":".join([ str(item) for item in [ int(pool.pool.svc.rl_ranks[index]) for index in range(pool.pool.svc.rl_nr) ] ]) self.daos_svcl.update(svcl, "daos_svcl" if display else None) def get_aggregate_total(self, processes): """Get the total bytes expected to be written by ior. Args: processes (int): number of processes running the ior command Returns: int: total number of bytes written """ power = {"k": 1, "m": 2, "g": 3, "t": 4} total = processes for name in ("block_size", "segment_count"): item = getattr(self, name).value if item: sub_item = re.split(r"([^\d])", str(item)) if sub_item > 0: total *= int(sub_item[0]) if len(sub_item) > 1: key = sub_item[1].lower() if key in power: total *= 1024**power[key] else: raise IorFailed( "Error obtaining the IOR aggregate total from " "the {} - bad key: value: {}, split: {}, " "key: {}".format(name, item, sub_item, key)) else: raise IorFailed( "Error obtaining the IOR aggregate total from the {}: " "value: {}, split: {}".format(name, item, sub_item)) # Account for any replicas try: # Extract the replica quantity from the object class string replica_qty = int(re.findall(r"\d+", self.daos_oclass.value)[0]) except (TypeError, IndexError): # If the daos object class is undefined (TypeError) or it does not # contain any numbers (IndexError) then there is only one replica replica_qty = 1 finally: total *= replica_qty return total def get_launch_command(self, manager, attach_info, processes, hostfile): """Get the process launch command used to run IOR. Args: manager (str): mpi job manager command attach_info (str): CART attach info path mpi_prefix (str): path for the mpi launch command processes (int): number of host processes hostfile (str): file defining host names and slots Raises: IorFailed: if an error occured building the IOR command Returns: str: ior launch command """ print("Getting launch command for {}".format(manager)) exports = "" env = { "CRT_ATTACH_INFO_PATH": attach_info, "MPI_LIB": "\"\"", "DAOS_SINGLETON_CLI": 1, } if manager.endswith("mpirun"): env.update({ "DAOS_POOL": self.daos_pool.value, "DAOS_SVCL": self.daos_svcl.value, "FI_PSM2_DISCONNECT": 1, "IOR_HINT__MPI__romio_daos_obj_class": self.mpiio_oclass, }) assign_env = ["{}={}".format(key, val) for key, val in env.items()] exports = "export {}; ".format("; export ".join(assign_env)) args = [ "-np {}".format(processes), "-hostfile {}".format(hostfile), # "-map-by node", ] elif manager.endswith("orterun"): assign_env = ["{}={}".format(key, val) for key, val in env.items()] args = [ "-np {}".format(processes), "-hostfile {}".format(hostfile), "-map-by node", ] args.extend(["-x {}".format(assign) for assign in assign_env]) elif manager.endswith("srun"): env.update({ "DAOS_POOL": self.daos_pool.value, "DAOS_SVCL": self.daos_svcl.value, "FI_PSM2_DISCONNECT": 1, }) assign_env = ["{}={}".format(key, val) for key, val in env.items()] args = [ "-l", "--mpi=pmi2", "--export={}".format(",".join(["ALL"] + assign_env)), ] if processes is not None: args.append("--ntasks={}".format(processes)) args.append("--distribution=cyclic") # --map-by node if hostfile is not None: args.append("--nodefile={}".format(hostfile)) else: raise IorFailed("Unsupported job manager: {}".format(manager)) return "{}{} {} {}".format(exports, manager, " ".join(args), self.__str__()) def run(self, manager, attach_info, processes, hostfile, display=True): """Run the IOR command. Args: manager (str): mpi job manager command attach_info (str): CART attach info path processes (int): number of host processes hostfile (str): file defining host names and slots display (bool, optional): print IOR output to the console. Defaults to True. Raises: IorFailed: if an error occured runnig the IOR command """ command = self.get_launch_command(manager, attach_info, processes, hostfile) if display: print("<IOR CMD>: {}".format(command)) # Run IOR try: run(command, allow_output_check="combined", shell=True) except CmdError as error: print("<IorRunFailed> Exception occurred: {}".format(error)) raise IorFailed("IOR Run process Failed: {}".format(error))