class DfuseCommand(ExecutableCommand): """Defines a object representing a dfuse command.""" def __init__(self, namespace, command): """Create a dfuse Command object.""" super().__init__(namespace, command) # dfuse options self.puuid = FormattedParameter("--pool {}") self.cuuid = FormattedParameter("--container {}") self.mount_dir = FormattedParameter("--mountpoint {}") self.sys_name = FormattedParameter("--sys-name {}") self.singlethreaded = FormattedParameter("--singlethread", False) self.foreground = FormattedParameter("--foreground", False) self.enable_caching = FormattedParameter("--enable-caching", False) self.enable_wb_cache = FormattedParameter("--enable-wb-cache", False) self.disable_caching = FormattedParameter("--disable-caching", False) self.disable_wb_cache = FormattedParameter("--disable-wb-cache", False) # Environment variable names to export when running dfuse self.update_env_names(["D_LOG_FILE"]) def set_dfuse_params(self, pool, display=True): """Set the dfuse params for the DAOS group, pool, and container uuid. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ self.set_dfuse_pool_params(pool, display) def set_dfuse_pool_params(self, pool, display=True): """Set Dfuse params based on Daos Pool. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ self.puuid.update(pool.uuid, "puuid" if display else None) def set_dfuse_cont_param(self, cont, display=True): """Set dfuse cont param from Container object. Args: cont (TestContainer): Daos test container object display (bool, optional): print updated params. Defaults to True. """ self.cuuid.update(cont, "cuuid" if display else None) def set_dfuse_exports(self, manager, log_file): """Set exports to issue before the dfuse command. Args: manager (DaosServerManager): server manager object to use to obtain the ofi and cart environmental variable settings from the server yaml file log_file (str): name of the log file to combine with the DAOS_TEST_LOG_DIR path with which to assign D_LOG_FILE """ env = self.get_environment(manager, log_file) self.set_environment(env)
class DserializeCommand(ExecutableCommand): """Defines an object representing a daos-serialize command.""" def __init__(self, namespace, command): """Create a daos-serialize Command object.""" super().__init__(namespace, command) # daos-serialize options # path to output serialized hdf5 files self.output_path = FormattedParameter("--output-path {}") # verbose output self.verbose = FormattedParameter("--verbose", False) # quiet output self.quiet = FormattedParameter("--quiet", False) # print help/usage self.print_usage = FormattedParameter("--help", False) # source path self.src_path = BasicParameter(None) def get_param_names(self): """Overriding the original get_param_names.""" param_names = super().get_param_names() # move key=src_path to the end param_names.sort(key='src_path'.__eq__) return param_names def set_dserialize_params(self, src_path=None, out_path=None, display=True): """Set common daos-serialize params. Args: src_path (str, optional): The source path formatted as daos://<pool>/<cont> out_path (str, optional): The output POSIX path to store the HDF5 file(s) display (bool, optional): print updated params. Defaults to True. """ if src_path: self.src_path.update(src_path, "src_path" if display else None) if out_path: self.output_path.update(out_path, "output_path" if display else None)
class DdeserializeCommand(ExecutableCommand): """Defines an object representing a daos-deserialize command.""" def __init__(self, namespace, command): """Create a daos-deserialize Command object.""" super().__init__(namespace, command) # daos-deserialize options # pool uuid for containers self.pool = FormattedParameter("--pool {}") # verbose output self.verbose = FormattedParameter("--verbose", False) # quiet output self.quiet = FormattedParameter("--quiet", False) # print help/usage self.print_usage = FormattedParameter("--help", False) # source path self.src_path = BasicParameter(None) def get_param_names(self): """Overriding the original get_param_names.""" param_names = super().get_param_names() # move key=src_path to the end param_names.sort(key='src_path'.__eq__) return param_names def set_ddeserialize_params(self, src_path=None, pool=None, display=True): """Set common daos-deserialize params. Args: src_path (str, optional): Either a list of paths to each HDF5 file, or the path to the directory containing the file(s). pool (str, optional): The pool uuid. display (bool, optional): print updated params. Defaults to True. """ if src_path: self.src_path.update(src_path, "src_path" if display else None) if pool: self.pool.update(pool, "pool" if display else None)
class DmgCommandBase(YamlCommand): """Defines a base object representing a dmg command.""" def __init__(self, path, yaml_cfg=None): """Create a dmg Command object. Args: path (str): path to the dmg command yaml_cfg (DmgYamlParameters, optional): dmg config file settings. Defaults to None, in which case settings must be supplied as command-line parameters. """ super().__init__("/run/dmg/*", "dmg", path, yaml_cfg) # If running dmg on remote hosts, this list needs to include those hosts self.temporary_file_hosts = gethostname().split(".")[0:1] # If specified use the configuration file from the YamlParameters object default_yaml_file = None if self.yaml is not None and hasattr(self.yaml, "filename"): default_yaml_file = self.yaml.filename self._hostlist = FormattedParameter("-l {}") self.hostfile = FormattedParameter("-f {}") self.configpath = FormattedParameter("-o {}", default_yaml_file) self.insecure = FormattedParameter("-i", False) self.debug = FormattedParameter("-d", True) self.json = FormattedParameter("-j", False) @property def hostlist(self): """Get the hostlist that was set. Returns a string list. """ if self.yaml: hosts = self.yaml.hostlist.value else: hosts = self._hostlist.value.split(",") return hosts @hostlist.setter def hostlist(self, hostlist): """Set the hostlist to be used for dmg invocation. Args: hostlist (string list): list of host addresses """ if self.yaml: if not isinstance(hostlist, list): hostlist = hostlist.split(",") self.yaml.hostlist.update(hostlist, "dmg.yaml.hostlist") else: if isinstance(hostlist, list): hostlist = ",".join(hostlist) self._hostlist.update(hostlist, "dmg._hostlist") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg sub command object based upon the sub-command.""" if self.sub_command.value == "network": self.sub_command_class = self.NetworkSubCommand() elif self.sub_command.value == "pool": self.sub_command_class = self.PoolSubCommand() elif self.sub_command.value == "storage": self.sub_command_class = self.StorageSubCommand() elif self.sub_command.value == "system": self.sub_command_class = self.SystemSubCommand() elif self.sub_command.value == "cont": self.sub_command_class = self.ContSubCommand() elif self.sub_command.value == "config": self.sub_command_class = self.ConfigSubCommand() elif self.sub_command.value == "telemetry": self.sub_command_class = self.TelemetrySubCommand() else: self.sub_command_class = None class ConfigSubCommand(CommandWithSubCommand): """Defines an object for the dmg config sub command.""" def __init__(self): """Create a dmg config subcommand object.""" super(DmgCommandBase.ConfigSubCommand, self).__init__("run/dmg/config/*", "config") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg config sub command object.""" if self.sub_command.value == "generate": self.sub_command_class = self.GenerateSubCommand() else: self.sub_command_class = None class GenerateSubCommand(CommandWithParameters): """Defines an object for the dmg config generate command.""" def __init__(self): """Create a dmg config generate object.""" super(DmgCommandBase.ConfigSubCommand.GenerateSubCommand, self).__init__("/run/dmg/config/generate/*", "generate") self.access_points = FormattedParameter( "--access-points={}", None) self.num_engines = FormattedParameter("--num-engines={}", None) self.min_ssds = FormattedParameter("--min-ssds={}", None) self.net_class = FormattedParameter("--net-class={}", None) class ContSubCommand(CommandWithSubCommand): """Defines an object for the dmg cont sub command.""" def __init__(self): """Create a dmg cont subcommand object.""" super().__init__("/run/dmg/cont/*", "cont") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg cont sub command object.""" if self.sub_command.value == "set-owner": self.sub_command_class = self.SetownerSubCommand() else: self.sub_command_class = None class SetownerSubCommand(CommandWithParameters): """Defines an object for the dmg cont set-owner command.""" def __init__(self): """Create a dmg cont set-owner command object.""" super().__init__("/run/dmg/cont/set-owner/*", "set-owner") self.pool = FormattedParameter("--pool={}", None) self.cont = FormattedParameter("--cont={}", None) self.user = FormattedParameter("--user={}", None) self.group = FormattedParameter("--group={}", None) class NetworkSubCommand(CommandWithSubCommand): """Defines an object for the dmg network sub command.""" def __init__(self): """Create a dmg network subcommand object.""" super().__init__("/run/dmg/network/*", "network") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg network sub command object.""" if self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() else: self.sub_command_class = None class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg network scan command.""" def __init__(self): """Create a dmg network scan command object.""" super().__init__("/run/dmg/network/scan/*", "scan") self.provider = FormattedParameter("-p {}", None) class PoolSubCommand(CommandWithSubCommand): """Defines an object for the dmg pool sub command.""" def __init__(self): """Create a dmg pool subcommand object.""" super().__init__("/run/dmg/pool/*", "pool") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg pool sub command object.""" if self.sub_command.value == "create": self.sub_command_class = self.CreateSubCommand() elif self.sub_command.value == "delete-acl": self.sub_command_class = self.DeleteAclSubCommand() elif self.sub_command.value == "destroy": self.sub_command_class = self.DestroySubCommand() elif self.sub_command.value == "get-acl": self.sub_command_class = self.GetAclSubCommand() elif self.sub_command.value == "list": self.sub_command_class = self.ListSubCommand() elif self.sub_command.value == "overwrite-acl": self.sub_command_class = self.OverwriteAclSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "set-prop": self.sub_command_class = self.SetPropSubCommand() elif self.sub_command.value == "update-acl": self.sub_command_class = self.UpdateAclSubCommand() elif self.sub_command.value == "exclude": self.sub_command_class = self.ExcludeSubCommand() elif self.sub_command.value == "extend": self.sub_command_class = self.ExtendSubCommand() elif self.sub_command.value == "drain": self.sub_command_class = self.DrainSubCommand() elif self.sub_command.value == "reintegrate": self.sub_command_class = self.ReintegrateSubCommand() elif self.sub_command.value == "evict": self.sub_command_class = self.EvictSubCommand() else: self.sub_command_class = None class CreateSubCommand(CommandWithParameters): """Defines an object for the dmg pool create command.""" def __init__(self): """Create a dmg pool create command object.""" super().__init__("/run/dmg/pool/create/*", "create") self.group = FormattedParameter("--group={}", None) self.user = FormattedParameter("--user={}", None) self.acl_file = FormattedParameter("--acl-file={}", None) self.size = FormattedParameter("--size={}", None) self.tier_ratio = FormattedParameter("--tier-ratio={}", None) self.scm_size = FormattedParameter("--scm-size={}", None) self.nvme_size = FormattedParameter("--nvme-size={}", None) self.ranks = FormattedParameter("--ranks={}", None) self.nsvc = FormattedParameter("--nsvc={}", None) self.sys = FormattedParameter("--sys={}", None) self.properties = FormattedParameter("--properties={}", None) self.label = FormattedParameter("--label={}", None) self.nranks = FormattedParameter("--nranks={}", None) class ExcludeSubCommand(CommandWithParameters): """Defines an object for the dmg pool exclude command.""" def __init__(self): """Create a dmg pool exclude command object.""" super().__init__("/run/dmg/pool/exclude/*", "exclude") self.pool = FormattedParameter("{}", None) self.rank = FormattedParameter("--rank={}", None) self.tgt_idx = FormattedParameter("--target-idx={}", None) class ExtendSubCommand(CommandWithParameters): """Defines an object for the dmg pool extend command.""" def __init__(self): """Create a dmg pool extend command object.""" super().__init__("/run/dmg/pool/extend/*", "extend") self.pool = FormattedParameter("{}", None) self.ranks = FormattedParameter("--ranks={}", None) class DrainSubCommand(CommandWithParameters): """Defines an object for the dmg pool drain command.""" def __init__(self): """Create a dmg pool drain command object.""" super().__init__("/run/dmg/pool/drain/*", "drain") self.pool = FormattedParameter("{}", None) self.rank = FormattedParameter("--rank={}", None) self.tgt_idx = FormattedParameter("--target-idx={}", None) class ReintegrateSubCommand(CommandWithParameters): """Defines an object for dmg pool reintegrate command.""" def __init__(self): """Create a dmg pool reintegrate command object.""" super().__init__("/run/dmg/pool/reintegrate/*", "reintegrate") self.pool = FormattedParameter("{}", None) self.rank = FormattedParameter("--rank={}", None) self.tgt_idx = FormattedParameter("--target-idx={}", None) class DeleteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool delete-acl command.""" def __init__(self): """Create a dmg pool delete-acl command object.""" super().__init__("/run/dmg/pool/delete-acl/*", "delete-acl") self.pool = FormattedParameter("{}", None) self.principal = FormattedParameter("-p {}", None) class DestroySubCommand(CommandWithParameters): """Defines an object for the dmg pool destroy command.""" def __init__(self): """Create a dmg pool destroy command object.""" super().__init__("/run/dmg/pool/destroy/*", "destroy") self.pool = FormattedParameter("{}", None) self.sys_name = FormattedParameter("--sys-name={}", None) self.force = FormattedParameter("--force", False) class GetAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool get-acl command.""" def __init__(self): """Create a dmg pool get-acl command object.""" super().__init__("/run/dmg/pool/get-acl/*", "get-acl") self.pool = FormattedParameter("{}", None) class ListSubCommand(CommandWithParameters): """Defines an object for the dmg pool list command.""" def __init__(self): """Create a dmg pool list command object.""" super().__init__("/run/dmg/pool/list/*", "list") self.no_query = FormattedParameter("--no-query", False) self.verbose = FormattedParameter("--verbose", False) class OverwriteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool overwrite-acl command.""" def __init__(self): """Create a dmg pool overwrite-acl command object.""" super().__init__("/run/dmg/pool/overwrite-acl/*", "overwrite-acl") self.pool = FormattedParameter("{}", None) self.acl_file = FormattedParameter("-a {}", None) class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg pool query command.""" def __init__(self): """Create a dmg pool query command object.""" super().__init__("/run/dmg/pool/query/*", "query") self.pool = FormattedParameter("{}", None) class SetPropSubCommand(CommandWithParameters): """Defines an object for the dmg pool set-prop command.""" def __init__(self): """Create a dmg pool set-prop command object.""" super().__init__("/run/dmg/pool/set-prop/*", "set-prop") self.pool = FormattedParameter("{}", None) self.name = FormattedParameter("--name={}", None) self.value = FormattedParameter("--value={}", None) class UpdateAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool update-acl command.""" def __init__(self): """Create a dmg pool update-acl command object.""" super().__init__("/run/dmg/pool/update-acl/*", "update-acl") self.pool = FormattedParameter("{}", None) self.acl_file = FormattedParameter("-a {}", None) self.entry = FormattedParameter("-e {}", None) class EvictSubCommand(CommandWithParameters): """Defines an object for the dmg pool evict command.""" def __init__(self): """Create a dmg pool evict command object.""" super().__init__("/run/dmg/pool/evict/*", "evict") self.pool = FormattedParameter("{}", None) self.sys = FormattedParameter("--sys={}", None) class StorageSubCommand(CommandWithSubCommand): """Defines an object for the dmg storage sub command.""" def __init__(self): """Create a dmg storage subcommand object.""" super().__init__("/run/dmg/storage/*", "storage") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg storage sub command object.""" if self.sub_command.value == "format": self.sub_command_class = self.FormatSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() elif self.sub_command.value == "set": self.sub_command_class = self.SetSubCommand() else: self.sub_command_class = None class FormatSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage format command object.""" super().__init__("/run/dmg/storage/format/*", "format") self.verbose = FormattedParameter("--verbose", False) self.reformat = FormattedParameter("--reformat", False) self.force = FormattedParameter("--force", False) class QuerySubCommand(CommandWithSubCommand): """Defines an object for the dmg storage query command.""" def __init__(self): """Create a dmg storage query command object.""" super().__init__("/run/dmg/storage/query/*", "query") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg storage query sub command object.""" if self.sub_command.value == "target-health": self.sub_command_class = self.TargetHealthSubCommand() elif self.sub_command.value == "device-health": self.sub_command_class = self.DeviceHealthSubCommand() elif self.sub_command.value == "list-devices": self.sub_command_class = self.ListDevicesSubCommand() elif self.sub_command.value == "list-pools": self.sub_command_class = self.ListPoolsSubCommand() else: self.sub_command_class = None class TargetHealthSubCommand(CommandWithParameters): """Defines a dmg storage query target-health object.""" def __init__(self): """Create a dmg storage query target-health object.""" super().__init__("/run/dmg/storage/query/target-health/*", "target-health") self.rank = FormattedParameter("-r {}", None) self.tgtid = FormattedParameter("-t {}", None) class DeviceHealthSubCommand(CommandWithParameters): """Defines a dmg storage query device-health object.""" def __init__(self): """Create a dmg storage query device-health object.""" super().__init__("/run/dmg/storage/query/device-health/*", "device-health") self.uuid = FormattedParameter("-u {}", None) class ListDevicesSubCommand(CommandWithParameters): """Defines a dmg storage query list-devices object.""" def __init__(self): """Create a dmg storage query list-devices object.""" super().__init__("/run/dmg/storage/query/list-devices/*", "list-devices") self.rank = FormattedParameter("-r {}", None) self.uuid = FormattedParameter("-u {}", None) self.health = FormattedParameter("-b", False) class ListPoolsSubCommand(CommandWithParameters): """Defines a dmg storage query list-pools object.""" def __init__(self): """Create a dmg storage query list-pools object.""" super().__init__("/run/dmg/storage/query/list-pools/*", "list-pools") self.rank = FormattedParameter("-r {}", None) self.uuid = FormattedParameter("-u {}", None) self.verbose = FormattedParameter("--verbose", False) class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg storage scan command.""" def __init__(self): """Create a dmg storage scan command object.""" super().__init__("/run/dmg/storage/scan/*", "scan") self.nvme_health = FormattedParameter("--nvme-health", False) self.verbose = FormattedParameter("--verbose", False) class SetSubCommand(CommandWithSubCommand): """Defines an object for the dmg storage set command.""" def __init__(self): """Create a dmg storage set command object.""" super().__init__("/run/dmg/storage/set/*", "set") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg set sub command object.""" if self.sub_command.value == "nvme-faulty": self.sub_command_class = self.NvmeFaultySubCommand() else: self.sub_command_class = None class NvmeFaultySubCommand(CommandWithParameters): """Defines a dmg storage set nvme-faulty object.""" def __init__(self): """Create a dmg storage set nvme-faulty object.""" super().__init__("/run/dmg/storage/query/device-state/*", "nvme-faulty") self.uuid = FormattedParameter("-u {}", None) self.force = FormattedParameter("--force", False) class SystemSubCommand(CommandWithSubCommand): """Defines an object for the dmg system sub command.""" def __init__(self): """Create a dmg system subcommand object.""" super().__init__("/run/dmg/system/*", "system") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg system sub command object.""" if self.sub_command.value == "leader-query": self.sub_command_class = self.LeaderQuerySubCommand() elif self.sub_command.value == "list-pools": self.sub_command_class = self.ListPoolsSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "start": self.sub_command_class = self.StartSubCommand() elif self.sub_command.value == "stop": self.sub_command_class = self.StopSubCommand() elif self.sub_command.value == "erase": self.sub_command_class = self.EraseSubCommand() elif self.sub_command.value == "cleanup": self.sub_command_class = self.CleanupSubCommand() else: self.sub_command_class = None class LeaderQuerySubCommand(CommandWithParameters): """Defines an object for the dmg system leader-query command.""" def __init__(self): """Create a dmg system leader-query command object.""" super().__init__("/run/dmg/system/leader-query/*", "leader-query") class ListPoolsSubCommand(CommandWithParameters): """Defines an object for the dmg system list-pools command.""" def __init__(self): """Create a dmg system list-pools command object.""" super().__init__("/run/dmg/system/list-pools/*", "list-pools") class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg system query command.""" def __init__(self): """Create a dmg system query command object.""" super().__init__("/run/dmg/system/query/*", "query") self.ranks = FormattedParameter("--ranks={}") self.verbose = FormattedParameter("--verbose", False) class CleanupSubCommand(CommandWithParameters): """Defines an object for the dmg system cleanup command.""" def __init__(self): """Create a dmg system cleanup command object.""" super().__init__("/run/dmg/system/cleanup/*", "cleanup") self.machinename = FormattedParameter("{}", None) self.verbose = FormattedParameter("--verbose", False) class StartSubCommand(CommandWithParameters): """Defines an object for the dmg system start command.""" def __init__(self): """Create a dmg system start command object.""" super().__init__("/run/dmg/system/start/*", "start") self.ranks = FormattedParameter("--ranks={}") self.rank_hosts = FormattedParameter("--rank-hosts={}") class StopSubCommand(CommandWithParameters): """Defines an object for the dmg system stop command.""" def __init__(self): """Create a dmg system stop command object.""" super().__init__("/run/dmg/system/stop/*", "stop") self.force = FormattedParameter("--force", False) self.ranks = FormattedParameter("--ranks={}") class EraseSubCommand(CommandWithParameters): """Defines an object for the dmg system erase command.""" def __init__(self): """Create a dmg system erase command object.""" super().__init__("/run/dmg/system/erase/*", "erase") class TelemetrySubCommand(CommandWithSubCommand): """Defines an object for the dmg telemetry sub command.""" def __init__(self): """Create a dmg telemetry subcommand object.""" super().__init__("/run/dmg/telemetry/*", "telemetry") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg telemetry sub command object.""" if self.sub_command.value == "metrics": self.sub_command_class = self.MetricsSubCommand() else: self.sub_command_class = None class MetricsSubCommand(CommandWithSubCommand): """Defines an object for the dmg telemetry metrics command.""" def __init__(self): """Create a dmg telemetry metrics command object.""" super().__init__("/run/dmg/telemetry/metrics/*", "metrics") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg telemetry metrics sub command object.""" if self.sub_command.value == "list": self.sub_command_class = self.ListSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() else: self.sub_command_class = None class ListSubCommand(CommandWithParameters): """Defines a dmg telemetry metrics list object.""" def __init__(self): """Create a dmg telemetry metrics list object.""" super().__init__("/run/dmg/telemetry/metrics/list/*", "list") self.host = FormattedParameter("--host-list={}", None) self.port = FormattedParameter("--port={}", None) class QuerySubCommand(CommandWithParameters): """Defines a dmg telemetry metrics query object.""" def __init__(self): """Create a dmg telemetry metrics query object.""" super().__init__("/run/dmg/telemetry/metrics/query/*", "query") self.host = FormattedParameter("--host-list={}", None) self.port = FormattedParameter("--port={}", None) self.metrics = FormattedParameter("--metrics={}", None)
class MacsioCommand(ExecutableCommand): # pylint: disable=too-many-instance-attributes """Defines an object from running the macsio command. Multi-purpose, Application-Centric, Scalable I/O Proxy Application https://github.com/LLNL/MACSio """ def __init__(self, path=""): """Create an MacsioCommand object. Args: path (str, optional): path to the macsio command. Defaults to "". """ super(MacsioCommand, self).__init__("/run/macsio/*", "macsio", path) # MACSio command parameters - defaults specified in square brackets: # --units_prefix_system %s ["binary"] # Specify which SI units prefix system to use both in reporting # performance data and in interpreting sizing modifiers to # arguments. The options are "binary" and "decimal". For "binary" # unit prefixes, sizes are reported in powers of 1024 and unit # symbols Ki, Mi, Gi, Ti, Pi are used. For "decimal", sizes are # reported in powers of 1000 and unit symbols are Kb, Mb, Gb, Tb, # Pb. See http://en.wikipedia.org/wiki/Binary_prefix. for more # information self.units_prefix_system = FormattedParameter( "--units_prefix_system {}") # --interface %s [miftmpl] # Specify the name of the interface to be tested. Use keyword # 'list' to print a list of all known interface names and then # exit. self.interface = FormattedParameter("--interface {}", "hdf5") # --parallel_file_mode %s %d [MIF 4] # Specify the parallel file mode. There are several choices. Use # 'MIF' for Multiple Independent File (Poor Man's) mode and then # also specify the number of files. Or, use 'MIFFPP' for MIF mode # and one file per processor or 'MIFOPT' for MIF mode and let the # test determine the optimum file count. Use 'SIF' for SIngle # shared File (Rich Man's) mode. If you also give a file count for # SIF mode, then MACSio will perform a sort of hybrid combination # of MIF and SIF modes. It will produce the specified number of # files by grouping ranks in the the same way MIF does, but I/O # within each group will be to a single, shared file using SIF # mode. # # Run macsio with SIF mode. MIF mode uses the HDF5 posix driver, # so it won't go through MPI-IO and hence not through the MPI-IO # DAOS driver. # # Note: Value should be specified as a string of a space- # separated string and integer value, e.g. 'SIF 1'. self.parallel_file_mode = FormattedParameter("--parallel_file_mode {}", "SIF 1") # --avg_num_parts %f [1] # The average number of mesh parts per MPI rank. Non-integral # values are acceptable. For example, a value that is half-way # between two integers, K and K+1, means that half the ranks have # K mesh parts and half have K+1 mesh parts. As another example, # a value of 2.75 here would mean that 75% of the ranks get 3 # parts and 25% of the ranks get 2 parts. Note that the total # number of parts is this number multiplied by the MPI # communicator size. If the result of that product is # non-integral, it will be rounded and a warning message will be # generated. self.avg_num_parts = FormattedParameter("--avg_num_parts {}") # --mesh_decomp %d %d %d [] # The layout of parts in the mesh overriding the simple # decomposition e.g. 4 8 1 will decompose into 32 parts in the # structure (x y z). # # Note: Value should be specified as a string of three space- # separated integer values, e.g. '4 8 1'. self.mesh_decomp = FormattedParameter("--mesh_decomp {}") # --part_size %d [80000] # Mesh part size in bytes. This becomes the nominal I/O request # size used by each MPI rank when marshalling data. A following # B|K|M|G character indicates 'B'ytes, 'K'ilo-, 'M'ega- or 'G'iga- # bytes representing powers of either 1000 or 1024 according to # the selected units prefix system. With no size modifier # character, 'B' is assumed. Mesh and variable data is then sized # by MACSio to hit this target byte count. However, due to # constraints involved in creating valid mesh topology and # variable data with realistic variation in features (e.g. zone- # and node-centering), this target byte count is hit exactly for # only the most frequently dumped objects and approximately for # other objects. self.part_size = FormattedParameter("--part_size {}") # --part_mesh_dims %d %d %d [] # Specify the number of elements in each dimension per mesh part. # This overrides the part_size parameter and instead allows the # size of the mesh to be determined by dimensions. e.g. 300 300 2, # 300 300 0 (set final dimension to 0 for 2d # # Note: Value should be specified as a string of three space- # separated integer values, e.g. '300 300 2'. self.part_mesh_dims = FormattedParameter("--part_mesh_dims {}") # --part_dim %d [2] # Spatial dimension of parts; 1, 2, or 3 self.part_dim = FormattedParameter("--part_dim {}") # --part_type %s [rectilinear] # Options are 'uniform', 'rectilinear', 'curvilinear', # 'unstructured' and 'arbitrary' (currently, only rectilinear is # implemented) self.part_type = FormattedParameter("--part_type {}") # --part_map %s [] # Specify the name of an ascii file containing part assignments to # MPI ranks. The ith line in the file, numbered from 0, holds the # MPI rank to which the ith part is to be assigned. (currently # ignored) self.part_map = FormattedParameter("--part_map {}") # --vars_per_part %d [20] # Number of mesh variable objects in each part. The smallest this # can be depends on the mesh type. For rectilinear mesh it is 1. # For curvilinear mesh it is the number of spatial dimensions and # for unstructured mesh it is the number of spatial dimensions # plus 2^number of topological dimensions. self.vars_per_part = FormattedParameter("--vars_per_part {}") # --dataset_growth %f [] # The factor by which the volume of data will grow between dump # iterations If no value is given or the value is <1.0 no dataset # changes will take place. self.dataset_growth = FormattedParameter("--dataset_growth {}") # --topology_change_probability %f [0.0] # The probability that the topology of the mesh (e.g. something # fundamental about the mesh's structure) will change between # dumps. A value of 1.0 indicates it should be changed every dump. # A value of 0.0, the default, indicates it will never change. A # value of 0.1 indicates it will change about once every 10 dumps. # Note: at present MACSio will not actually compute/construct a # different topology. It will only inform a plugin that a given # dump should be treated as a change in topology. self.topology_change_probability = FormattedParameter( "--topology_change_probability {}") # --meta_type %s [tabular] # Specify the type of metadata objects to include in each main # dump. Options are 'tabular', 'amorphous'. For tabular type # data, MACSio will generate a random set of tables of somewhat # random structure and content. For amorphous, MACSio will # generate a random hierarchy of random type and sized objects. self.meta_type = FormattedParameter("--meta_type {}") # --meta_size %d %d [10000 50000] # Specify the size of the metadata objects on each processor and # separately, the root (or master) processor (MPI rank 0). The # size is specified in terms of the total number of bytes in the # metadata objects MACSio creates. For example, a type of tabular # and a size of 10K bytes might result in 3 random tables; one # table with 250 unnamed records where each record is an array of # 3 doubles for a total of 6000 bytes, another table of 200 # records where each record is a named integer value where each # name is length 8 chars for a total of 2400 bytes and a 3rd table # of 40 unnamed records where each record is a 40 byte struct # comprised of ints and doubles for a total of 1600 bytes. # # Note: Value should be specified as a string of two space- # separated integer values, e.g. '10000 50000'. self.meta_size = FormattedParameter("--meta_size {}") # --num_dumps %d [10] # The total number of dumps to marshal. self.num_dumps = FormattedParameter("--num_dumps {}") # --max_dir_size %d [] # The maximum number of filesystem objects (e.g. files or # subdirectories) that MACSio will create in any one subdirectory. # This is typically relevant only in MIF mode because MIF mode can # wind up generating many will continue to create output files in # the same directory until it has completed all dumps. Use a value # of zero to force MACSio to put each dump in a separate directory # but where the number of top-level directories is still # unlimited. The result will be a 2-level directory hierarchy with # dump directories at the top and individual dump files in each # directory. A value > 0 will cause MACSio to create a tree-like # directory structure where the files are the leaves and # encompassing dir tree is created such as to maintain the # max_dir_size constraint specified here. For example, if the # value is set to 32 and the MIF file count is 1024, then each # dump will involve a 3-level dir-tree; the top dir containing 32 # sub-dirs and each sub-dir containing 32 of the 1024 files for # the dump. If more than 32 dumps are performed, then the dir-tree # will really be 4 or more levels with the first 32 dumps' # dir-trees going into the first dir, etc. self.max_dir_size = FormattedParameter("--max_dir_size {}") # --compute_work_intensity %d [1] # Add some work in between I/O phases. There are three levels of # 'compute' that can be performed as follows: # Level 1: Perform a basic sleep operation # Level 2: Perform some simple FLOPS with randomly accessed # data # Level 3: Solves the 2D Poisson equation via the Jacobi # iterative method # This input is intended to be used in conjunction with # --compute_time which will roughly control how much time is spent # doing work between iops self.compute_work_intensity = FormattedParameter( "--compute_work_intensity {}") # --compute_time %f [] # A rough lower bound on the number of seconds spent doing work # between I/O phases. The type of work done is controlled by the # --compute_work_intensity input and defaults to Level 1 (basic # sleep). self.compute_time = FormattedParameter("--compute_time {}") # --alignment %d [] # Not currently documented self.alignment = FormattedParameter("--alignment {}") # --filebase %s [macsio] # Basename of generated file(s). self.filebase = FormattedParameter("--filebase {}") # --fileext %s [] # Extension of generated file(s). self.fileext = FormattedParameter("--fileext {}") # --read_path %s [] # Specify a path name (file or dir) to start reading for a read # test. self.read_path = FormattedParameter("--read_path {}") # --num_loads %d [] # Number of loads in succession to test. self.num_loads = FormattedParameter("--num_loads {}") # --no_validate_read [] # Don't validate data on read. self.no_validate_read = FormattedParameter("--no_validate_read {}") # --read_mesh %s [] # Specify mesh name to read. self.read_mesh = FormattedParameter("--read_mesh {}") # --read_vars %s [] # Specify variable names to read. "all" means all variables. If # listing more than one, be sure to either enclose space separated # list in quotes or use a comma-separated list with no spaces self.read_vars = FormattedParameter("--read_vars {}") # --time_randomize [] # Make randomness in MACSio vary from dump to dump and run to run # by using PRNGs seeded by time. self.time_randomize = FormattedParameter("--time_randomize {}") # --plugin_args %n [] # All arguments after this sentinel are passed to the I/O plugin # plugin. The '%n' is a special designator for the builtin 'argi' # value. self.plugin_args = FormattedParameter("--plugin_args {}") # --debug_level %d [0] # Set debugging level (1, 2 or 3) of log files. Higher numbers # mean more frequent and detailed output. A value of zero, the # default, turns all debugging output off. A value of 1 should not # adversely effect performance. A value of 2 may effect # performance and a value of 3 will almost certainly effect # performance. For debug level 3, MACSio will generate ascii json # files from each processor for the main dump object prior to # starting dumps. self.debug_level = FormattedParameter("--debug_level {}") # # Log File Options to control size and shape of log file: # # --log_file_name %s [macsio-log.log] # The name of the log file. self.log_file_name = FormattedParameter("--log_file_name {}", "macsio-log.log") # --log_line_cnt %d %d [64 0] # Set number of lines per rank in the log file and number of extra # lines for rank 0. self.log_line_cnt = FormattedParameter("--log_line_cnt {}") # --log_line_length %d [128] # Set log file line length. self.log_line_length = FormattedParameter("--log_line_length {}") # --timings_file_name %s [macsio-timings.log] # Specify the name of the timings file. Passing an empty string, # "" will disable the creation of a timings file. self.timings_file_name = FormattedParameter("--timings_file_name {}", "macsio-timings.log") # # Options specific to the "hdf5" I/O plugin # # --show_errors [] # Show low-level HDF5 errors self.show_errors = FormattedParameter("--show_errors {}") # --compression %s %s [] # The first string argument is the compression algorithm name. The # second string argument is a comma-separated set of params of the # form 'param1=val1,param2=val2,param3=val3. The various algorithm # names and their parameter meanings are described below. Note # that some parameters are not specific to any algorithm. Those # are described first followed by individual algorithm-specific # parameters for those algorithms available in the current build. # # minsize=%d [1024] # minimum size of dataset (in terms of a count of values) # upon which compression will even be attempted # # shuffle=<int> # Boolean (zero or non-zero) to indicate whether to use # HDF5's byte shuffling filter *prior* to compression. # Default depends on algorithm. By default, shuffling is # NOT used for zfp but IS used with all other algorithms. # # Available compression algorithms: # # "zfp" # Use Peter Lindstrom's ZFP compression ( # computation.llnl.gov/casc/zfp) Note: Whether this # compression is available is determined entirely at # run-time using the H5Z-ZFP compressor as a generic # filter. This means all that is necessary is to specify # the HDF5_PLUGIN_PATH environment variable with a path # to the shared lib for the filter. # # The following ZFP options are *mutually*exclusive*. # In any command-line specifying more than one of the # following options, only the last specified will be # honored. # # rate=%f [] # target # bits per compressed output datum. # Fractional values are permitted. 0 selects defaults: # 4 bits/flt or 8 bits/dbl. Use this option to hit a # target compressed size but where error varies. OTOH, # use one of the following two options for fixed error # but amount of compression, if any, varies. # # precision=%d [] # # bits of precision to preserve in each input datum. # # accuracy=%f [] # absolute error tolerance in each output datum. In # many respects, 'precision' represents a sort of # relative error tolerance while 'accuracy' represents # an absolute tolerance. See # http://en.wikipedia.org/wiki/Accuracy_and_precision. # # "gzip" # level=%d [9] # A value in the range [1,9], inclusive, trading off # time to compress with amount of compression. Level=1 # results in best speed but worst compression whereas # level=9 results in best compression but worst speed. # Values outside [1,9] are clamped. # # Examples: # --compression zfp rate=18.5 # --compression gzip minsize=1024,level=9 # --compression szip shuffle=0,options=nn,pixels_per_block=16 self.compression = FormattedParameter("--compression {}") # --no_collective [] # Use independent, not collective, I/O calls in SIF mode. self.no_collective = FormattedParameter("--no_collective {}") # --no_single_chunk [] # Do not single chunk the datasets (currently ignored). self.no_single_chunk = FormattedParameter("--no_single_chunk {}") # --sieve_buf_size %d [] # Specify sieve buffer size (see H5Pset_sieve_buf_size) self.sieve_buf_size = FormattedParameter("--sieve_buf_size {}") # --meta_block_size %d [] # Specify size of meta data blocks (see H5Pset_meta_block_size) self.meta_block_size = FormattedParameter("--meta_block_size {}") # --small_block_size %d [] # Specify threshold size for data blocks considered to be 'small' # (see H5Pset_small_data_block_size) self.small_block_size = FormattedParameter("--small_block_size {}") # --log [] # Use logging Virtual File Driver (see H5Pset_fapl_log) self.log_virtual_file_driver = FormattedParameter("--log {}") # DAOS parameters self.daos_pool = None self.daos_svcl = None self.daos_cont = None # Environment variable names required to be set when running the macsio # command. The values for these names are populated by the # set_environment() method. self._env_names = ["D_LOG_FILE"] def set_output_file_path(self): """Set the path for the files generated by the macsio command.""" self.log_file_name.update(get_log_file(self.log_file_name.value), "macsio.log_file_name") self.timings_file_name.update( get_log_file(self.timings_file_name.value), "macsio.timings_file_name") def get_environment(self, manager, log_file=None): """Get the environment variables to export for the command. Args: manager (DaosServerManager): the job manager used to start daos_server from which the server config values can be obtained to set the required environment variables. log_file (str, optional): when specified overrides the default D_LOG_FILE value. Defaults to None. Returns: EnvironmentVariables: a dictionary of environment variable names and values to export prior to running daos_racer """ env = super(MacsioCommand, self).get_environment(manager, log_file) mapping = { "DAOS_POOL": self.daos_pool, "DAOS_SVCL": self.daos_svcl, "DAOS_CONT": self.daos_cont, } for key in mapping: if mapping[key]: env[key] = mapping[key] return env def check_results(self, result): """Check the macsio command results. Args: results (CmdResult): macsio command execution result Returns: bool: status of the macsio command results """ self.log.info("The '%s' command completed with exit status: %s", self.__str__(), result.exit_status) # Basic check of the macsio command status status = result.exit_status == 0 return status
class DsyncCommand(ExecutableCommand): """Defines an object representing a dsync command.""" def __init__(self, namespace, command): """Create a dsync Command object.""" super(DsyncCommand, self).__init__(namespace, command) # dsync options # show differences, but do not synchronize files self.dryrun = FormattedParameter("--dryrun", False) # batch files into groups of N during copy self.batch_files = FormattedParameter("--batch-files {}") # IO buffer size in bytes (default 4MB) self.bufsize = FormattedParameter("--blocksize {}") # work size per task in bytes (default 4MB) self.chunksize = FormattedParameter("--chunksize {}") # DAOS prefix for unified namespace path self.daos_prefix = FormattedParameter("--daos-prefix {}") # DAOS API in {DFS, DAOS} (default uses DFS for POSIX containers) self.daos_api = FormattedParameter("--daos-api {}") # read and compare file contents rather than compare size and mtime self.contents = FormattedParameter("--contents", False) # delete extraneous files from target self.delete = FormattedParameter("--delete", False) # copy original files instead of links self.dereference = FormattedParameter("--dereference", False) # don't follow links in source self.no_dereference = FormattedParameter("--no-dereference", False) # open files with O_DIRECT self.direct = FormattedParameter("--direct", False) # hardlink to files in DIR when unchanged self.link_dest = FormattedParameter("--link-dest {}") # create sparse files when possible self.sparse = FormattedParameter("--sparse", False) # print progress every N seconds self.progress = FormattedParameter("--progress {}") # verbose output self.verbose = FormattedParameter("--verbose", False) # quiet output self.quiet = FormattedParameter("--quiet", False) # print help/usage self.print_usage = FormattedParameter("--help", False) # source path self.src_path = BasicParameter(None) # destination path self.dst_path = BasicParameter(None) def get_param_names(self): """Overriding the original get_param_names.""" param_names = super(DsyncCommand, self).get_param_names() # move key=dst_path to the end param_names.sort(key='dst_path'.__eq__) return param_names def set_dsync_params(self, src=None, dst=None, prefix=None, display=True): """Set common dsync params. Args: src (str, optional): The source path formatted as daos://<pool>/<cont>/<path> or <path> dst (str, optional): The destination path formatted as daos://<pool>/<cont>/<path> or <path> prefix (str, optional): prefix for uns path display (bool, optional): print updated params. Defaults to True. """ if src: self.src_path.update(src, "src_path" if display else None) if dst: self.dst_path.update(dst, "dst_path" if display else None) if prefix: self.daos_prefix.update(prefix, "daos_prefix" if display else None)
class DcpCommand(ExecutableCommand): """Defines an object representing a dcp command.""" def __init__(self, namespace, command): """Create a dcp Command object.""" super(DcpCommand, self).__init__(namespace, command) # dcp options # IO buffer size in bytes (default 64MB) self.blocksize = FormattedParameter("--blocksize {}") # New versions use bufsize instead of blocksize self.bufsize = FormattedParameter("--bufsize {}") # work size per task in bytes (default 64MB) self.chunksize = FormattedParameter("--chunksize {}") # DAOS source pool self.daos_src_pool = FormattedParameter("--daos-src-pool {}") # DAOS destination pool self.daos_dst_pool = FormattedParameter("--daos-dst-pool {}") # DAOS source container self.daos_src_cont = FormattedParameter("--daos-src-cont {}") # DAOS destination container self.daos_dst_cont = FormattedParameter("--daos-dst-cont {}") # DAOS prefix for unified namespace path self.daos_prefix = FormattedParameter("--daos-prefix {}") # DAOS API in {DFS, DAOS} (default uses DFS for POSIX containers) self.daos_api = FormattedParameter("--daos-api {}") # read source list from file self.input_file = FormattedParameter("--input {}") # copy original files instead of links self.dereference = FormattedParameter("--dereference", False) # don't follow links in source self.no_dereference = FormattedParameter("--no-dereference", False) # preserve permissions, ownership, timestamps, extended attributes self.preserve = FormattedParameter("--preserve", False) # open files with O_DIRECT self.direct = FormattedParameter("--direct", False) # create sparse files when possible self.sparse = FormattedParameter("--sparse", False) # print progress every N seconds self.progress = FormattedParameter("--progress {}") # verbose output self.verbose = FormattedParameter("--verbose", False) # quiet output self.quiet = FormattedParameter("--quiet", False) # print help/usage self.print_usage = FormattedParameter("--help", False) # source path self.src_path = BasicParameter(None) # destination path self.dst_path = BasicParameter(None) def get_param_names(self): """Overriding the original get_param_names.""" param_names = super(DcpCommand, self).get_param_names() # move key=dst_path to the end param_names.sort(key='dst_path'.__eq__) return param_names def set_dcp_params(self, src_pool=None, src_cont=None, src_path=None, dst_pool=None, dst_cont=None, dst_path=None, prefix=None, display=True): """Set common dcp params. Args: src_pool (str, optional): source pool uuid src_cont (str, optional): source container uuid src_path (str, optional): source path dst_pool (str, optional): destination pool uuid dst_cont (str, optional): destination container uuid dst_path (str, optional): destination path prefix (str, optional): prefix for uns path display (bool, optional): print updated params. Defaults to True. """ if src_pool: self.daos_src_pool.update(src_pool, "daos_src_pool" if display else None) if src_cont: self.daos_src_cont.update(src_cont, "daos_src_cont" if display else None) if src_path: self.src_path.update(src_path, "src_path" if display else None) if dst_pool: self.daos_dst_pool.update(dst_pool, "daos_dst_pool" if display else None) if dst_cont: self.daos_dst_cont.update(dst_cont, "daos_dst_cont" if display else None) if dst_path: self.dst_path.update(dst_path, "dst_path" if display else None) if prefix: self.daos_prefix.update(prefix, "daos_prefix" if display else None)
class IorCommand(ExecutableCommand): # pylint: disable=too-many-instance-attributes """Defines a object for executing an IOR command. Example: >>> # Typical use inside of a DAOS avocado test method. >>> ior_cmd = IorCommand() >>> ior_cmd.get_params(self) >>> ior_cmd.set_daos_params(self.server_group, self.pool) >>> mpirun = Mpirun() >>> server_manager = self.server_manager[0] >>> env = self.ior_cmd.get_environment(server_manager, self.client_log) >>> mpirun.assign_hosts(self.hostlist_clients, self.workdir, None) >>> mpirun.assign_processes(len(self.hostlist_clients)) >>> mpirun.assign_environment(env) >>> mpirun.run() """ def __init__(self): """Create an IorCommand object.""" super().__init__("/run/ior/*", "ior") # Flags self.flags = FormattedParameter("{}") # Optional arguments # -a=POSIX API for I/O [POSIX|DUMMY|MPIIO|MMAP|DFS|HDF5] # -b=1048576 blockSize -- contiguous bytes to write per task # -d=0 interTestDelay -- delay between reps in seconds # -f=STRING scriptFile -- test script name # -G=0 setTimeStampSignature -- time stamp signature # -i=1 repetitions -- number of repetitions of test # -j=0 outlierThreshold -- warn on outlier N sec from mean # -J=1 setAlignment -- HDF5 alignment in bytes # -l=STRING datapacket type-- type of packet created # -M=STRING memoryPerNode -- hog memory on the node # -N=0 numTasks -- num of participating tasks in the test # -o=testFile testFile -- full name for test # -O=STRING string of IOR directives # -O=1 stoneWallingWearOut -- all process finish to access # the amount of data after stonewalling timeout # -O=0 stoneWallingWearOutIterations -- stop after # processing this number of iterations # -O=STRING stoneWallingStatusFile -- file to keep number of # iterations from stonewalling during write # -Q=1 taskPerNodeOffset for read tests # -s=1 segmentCount -- number of segments # -t=262144 transferSize -- size of transfer in bytes # -T=0 maxTimeDuration -- max time in minutes executing # repeated test; it aborts only between iterations # and not within a test! self.api = FormattedParameter("-a {}", "DFS") self.block_size = FormattedParameter("-b {}") self.test_delay = FormattedParameter("-d {}") self.script = FormattedParameter("-f {}") self.signature = FormattedParameter("-G {}") self.repetitions = FormattedParameter("-i {}") self.outlier_threshold = FormattedParameter("-j {}") self.alignment = FormattedParameter("-J {}") self.data_packet_type = FormattedParameter("-l {}") self.memory_per_node = FormattedParameter("-M {}") self.num_tasks = FormattedParameter("-N {}") self.test_file = FormattedParameter("-o {}") self.directives = FormattedParameter("-O {}") self.sw_wearout = FormattedParameter( "-O stoneWallingWearOut={}") self.sw_wearout_iteration = FormattedParameter( "-O stoneWallingWearOutIterations={}") self.sw_status_file = FormattedParameter( "-O stoneWallingStatusFile={}") self.task_offset = FormattedParameter("-Q {}") self.segment_count = FormattedParameter("-s {}") self.transfer_size = FormattedParameter("-t {}") self.max_duration = FormattedParameter("-T {}") # Module DFS # Required arguments # --dfs.pool=STRING pool uuid # --dfs.cont=STRING container uuid # Flags # --dfs.destroy Destroy Container # Optional arguments # --dfs.group=STRING server group # --dfs.chunk_size=1048576 chunk size # --dfs.oclass=STRING object class # --dfs.prefix=STRING mount prefix self.dfs_pool = FormattedParameter("--dfs.pool {}") self.dfs_cont = FormattedParameter("--dfs.cont {}") self.dfs_destroy = FormattedParameter("--dfs.destroy", False) self.dfs_group = FormattedParameter("--dfs.group {}") self.dfs_chunk = FormattedParameter("--dfs.chunk_size {}", 1048576) self.dfs_oclass = FormattedParameter("--dfs.oclass {}", "SX") self.dfs_dir_oclass = FormattedParameter("--dfs.dir_oclass {}", "SX") self.dfs_prefix = FormattedParameter("--dfs.prefix {}") # A list of environment variable names to set and export with ior self._env_names = ["D_LOG_FILE"] # Attributes used to determine command success when run as a subprocess # See self.check_ior_subprocess_status() for details. self.pattern = None self.pattern_count = 1 def get_param_names(self): """Get a sorted list of the defined IorCommand parameters.""" # Sort the IOR parameter names to generate consistent ior commands all_param_names = super().get_param_names() # List all of the common ior params first followed by any daos-specific # and dfs-specific params (except when using MPIIO). param_names = [name for name in all_param_names if ("daos" not in name) and ("dfs" not in name)] if self.api.value == "DFS": param_names.extend( [name for name in all_param_names if "dfs" in name]) return param_names def set_daos_params(self, group, pool, cont_uuid=None, display=True): """Set the IOR parameters for the DAOS group, pool, and container uuid. Args: group (str): DAOS server group name pool (TestPool): DAOS test pool object cont_uuid (str, optional): the container uuid. If not specified one is generated. Defaults to None. display (bool, optional): print updated params. Defaults to True. """ self.set_daos_pool_params(pool, display) if self.api.value in ["DFS", "MPIIO", "POSIX", "HDF5"]: self.dfs_group.update(group, "dfs_group" if display else None) self.dfs_cont.update( cont_uuid if cont_uuid else str(uuid.uuid4()), "dfs_cont" if display else None) def set_daos_pool_params(self, pool, display=True): """Set the IOR parameters that are based on a DAOS pool. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ if self.api.value in ["DFS", "MPIIO", "POSIX", "HDF5"]: self.dfs_pool.update( pool.pool.get_uuid_str(), "dfs_pool" if display else None) def get_aggregate_total(self, processes): """Get the total bytes expected to be written by ior. Args: processes (int): number of processes running the ior command Returns: int: total number of bytes written Raises: CommandFailure: if there is an error obtaining the aggregate total """ power = {"k": 1, "m": 2, "g": 3, "t": 4} total = processes for name in ("block_size", "segment_count"): item = getattr(self, name).value if item: sub_item = re.split(r"([^\d])", str(item)) if int(sub_item[0]) > 0: total *= int(sub_item[0]) if len(sub_item) > 1: key = sub_item[1].lower() if key in power: total *= 1024**power[key] else: raise CommandFailure( "Error obtaining the IOR aggregate total from " "the {} - bad key: value: {}, split: {}, " "key: {}".format(name, item, sub_item, key)) else: raise CommandFailure( "Error obtaining the IOR aggregate total from the {}: " "value: {}, split: {}".format(name, item, sub_item)) # Account for any replicas, except for the ones with no replication # i.e all object classes starting with "S". Eg: S1,S2,...,SX. if not self.dfs_oclass.value.startswith("S"): try: # Extract the replica quantity from the object class string replica_qty = int(re.findall(r"\d+", self.dfs_oclass.value)[0]) except (TypeError, IndexError): # If the daos object class is undefined (TypeError) or it does # not contain any numbers (IndexError) then there is only one # replica. replica_qty = 1 finally: total *= replica_qty return total def get_default_env(self, manager_cmd, log_file=None): """Get the default environment settings for running IOR. Args: manager_cmd (str): job manager command log_file (str, optional): log file. Defaults to None. Returns: EnvironmentVariables: a dictionary of environment names and values """ env = self.get_environment(None, log_file) env["MPI_LIB"] = "\"\"" env["FI_PSM2_DISCONNECT"] = "1" # ior POSIX api does not require the below options. if "POSIX" in manager_cmd: return env if "mpirun" in manager_cmd or "srun" in manager_cmd: if self.dfs_pool.value is not None: env["DAOS_UNS_PREFIX"] = "daos://{}/{}/".format(self.dfs_pool.value, self.dfs_cont.value) if self.dfs_oclass.value is not None: env["IOR_HINT__MPI__romio_daos_obj_class"] = \ self.dfs_oclass.value return env @staticmethod def get_ior_metrics(cmdresult): """Get the ior command read and write metrics. Parse the CmdResult (output of the test) and look for the ior stdout and get the read and write metrics. Args: cmdresult (CmdResult): output of job manager Returns: metrics (tuple) : list of write and read metrics from ior run """ ior_metric_summary = "Summary of all tests:" messages = cmdresult.stdout_text.splitlines() # Get the index whre the summary starts and add one to # get to the header. idx = messages.index(ior_metric_summary) # idx + 1 is header. # idx +2 and idx + 3 will give the write and read metrics. write_metrics = (" ".join(messages[idx+2].split())).split() read_metrics = (" ".join(messages[idx+3].split())).split() return (write_metrics, read_metrics) @staticmethod def log_metrics(logger, message, metrics): """Log the ior metrics. Args: logger (log): logger object handle message (str) : Message to print before logging metrics metric (lst) : IOR write and read metrics """ logger.info("\n") logger.info(message) for metric in metrics: logger.info(metric) logger.info("\n") def check_ior_subprocess_status(self, sub_process, command, pattern_timeout=10): """Verify the status of the command started as a subprocess. Continually search the subprocess output for a pattern (self.pattern) until the expected number of patterns (self.pattern_count) have been found (typically one per host) or the timeout (pattern_timeout) is reached or the process has stopped. Args: sub_process (process.SubProcess): subprocess used to run the command command (str): ior command being looked for pattern_timeout: (int): check pattern until this timeout limit is reached. Returns: bool: whether or not the command progress has been detected """ complete = True self.log.info( "Checking status of the %s command in %s with a %s second timeout", command, sub_process, pattern_timeout) if self.pattern is not None: detected = 0 complete = False timed_out = False start = time.time() # Search for patterns in the subprocess output until: # - the expected number of pattern matches are detected (success) # - the time out is reached (failure) # - the subprocess is no longer running (failure) while not complete and not timed_out and sub_process.poll() is None: output = get_subprocess_stdout(sub_process) detected = len(re.findall(self.pattern, output)) complete = detected == self.pattern_count timed_out = time.time() - start > pattern_timeout # Summarize results msg = "{}/{} '{}' messages detected in {}/{} seconds".format( detected, self.pattern_count, self.pattern, time.time() - start, pattern_timeout) if not complete: # Report the error / timeout self.log.info( "%s detected - %s:\n%s", "Time out" if timed_out else "Error", msg, get_subprocess_stdout(sub_process)) # Stop the timed out process if timed_out: self.stop() else: # Report the successful start self.log.info( "%s subprocess startup detected - %s", command, msg) return complete
class MdtestCommand(ExecutableCommand): """Defines a object representing a mdtest command.""" def __init__(self): """Create an MdtestCommand object.""" super(MdtestCommand, self).__init__("/run/mdtest/*", "mdtest") self.flags = FormattedParameter("{}") # mdtest flags # Optional arguments # -a=STRING API for I/O [POSIX|DUMMY] # -b=1 branching factor of hierarchical dir structure # -d=./out the directory in which the tests will run # -B=0 no barriers between phases # -e=0 bytes to read from each file # -f=1 first number of tasks on which test will run # -i=1 number of iterations the test will run # -I=0 number of items per directory in tree # -l=0 last number of tasks on which test will run # -n=0 every process will creat/stat/read/remove num # of directories and files # -N=0 stride num between neighbor tasks for file/dir # operation (local=0) # -p=0 pre-iteration delay (in seconds) # --random-seed=0 random seed for -R # -s=1 stride between number of tasks for each test # -V=0 verbosity value # -w=0 bytes to write each file after it is created # -W=0 number in seconds; stonewall timer, write as # many seconds and ensure all processes did the # same number of operations (currently only # stops during create phase) # -x=STRING StoneWallingStatusFile; contains the number # of iterations of the creation phase, can be # used to split phases across runs # -z=0 depth of hierarchical directory structure self.api = FormattedParameter("-a {}") self.branching_factor = FormattedParameter("-b {}") self.test_dir = FormattedParameter("-d {}") self.barriers = FormattedParameter("-B {}") self.read_bytes = FormattedParameter("-e {}") self.first_num_tasks = FormattedParameter("-f {}") self.iteration = FormattedParameter("-i {}") self.items = FormattedParameter("-I {}") self.last_num_tasks = FormattedParameter("-l {}") self.num_of_files_dirs = FormattedParameter("-n {}") self.pre_iter = FormattedParameter("-p {}") self.random_seed = FormattedParameter("--random-seed {}") self.stride = FormattedParameter("-s {}") self.verbosity_value = FormattedParameter("-V {}") self.write_bytes = FormattedParameter("-w {}") self.stonewall_timer = FormattedParameter("-W {}") self.stonewall_statusfile = FormattedParameter("-x {}") self.depth = FormattedParameter("-z {}") # Module DFS # Required arguments # --dfs.pool=STRING DAOS pool uuid # --dfs.svcl=STRING DAOS pool SVCL # --dfs.cont=STRING DFS container uuid # Flags # --dfs.destroy Destroy DFS Container # Optional arguments # --dfs.group=STRING DAOS server group # --dfs.chunk_size=1048576 Chunk size # --dfs.oclass=STRING DAOS object class # --dfs.dir_oclass=STRING DAOS directory object class # --dfs.prefix=STRING Mount prefix self.dfs_pool_uuid = FormattedParameter("--dfs.pool {}") self.dfs_svcl = FormattedParameter("--dfs.svcl {}") self.dfs_cont = FormattedParameter("--dfs.cont {}") self.dfs_group = FormattedParameter("--dfs.group {}") self.dfs_destroy = FormattedParameter("--dfs.destroy", True) self.dfs_chunk = FormattedParameter("--dfs.chunk_size {}", 1048576) self.dfs_oclass = FormattedParameter("--dfs.oclass {}", "SX") self.dfs_prefix = FormattedParameter("--dfs.prefix {}") self.dfs_dir_oclass = FormattedParameter("--dfs.dir_oclass {}", "SX") # A list of environment variable names to set and export with ior self._env_names = ["D_LOG_FILE"] def get_param_names(self): """Get a sorted list of the defined MdtestCommand parameters.""" # Sort the Mdtest parameter names to generate consistent ior commands all_param_names = super(MdtestCommand, self).get_param_names() # List all of the common ior params first followed by any dfs-specific # params (except when using POSIX). param_names = [name for name in all_param_names if "dfs" not in name] if self.api.value != "POSIX": param_names.extend( [name for name in all_param_names if "dfs" in name]) return param_names def set_daos_params(self, group, pool, cont_uuid=None, display=True): """Set the Mdtest params for the DAOS group, pool, and container uuid. Args: group (str): DAOS server group name pool (TestPool): DAOS test pool object cont_uuid (str, optional): the container uuid. If not specified one is generated. Defaults to None. display (bool, optional): print updated params. Defaults to True. """ self.set_daos_pool_params(pool, display) self.dfs_group.update(group, "dfs_group" if display else None) self.dfs_cont.update( cont_uuid if cont_uuid else str(uuid.uuid4()), "dfs_cont" if display else None) def set_daos_pool_params(self, pool, display=True): """Set the Mdtest parameters that are based on a DAOS pool. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ self.dfs_pool_uuid.update( pool.pool.get_uuid_str(), "dfs_pool" if display else None) self.set_daos_svcl_param(pool, display) def set_daos_svcl_param(self, pool, display=True): """Set the Mdtest daos_svcl param from the ranks of a DAOS pool object. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ svcl = ":".join( [str(item) for item in [ int(pool.pool.svc.rl_ranks[index]) for index in range(pool.pool.svc.rl_nr)]]) self.dfs_svcl.update(svcl, "dfs_svcl" if display else None) def get_default_env(self, manager_cmd, log_file=None): """Get the default environment settings for running mdtest. Args: manager_cmd (str): job manager command log_file (str, optional): log file. Defaults to None. Returns: EnvironmentVariables: a dictionary of environment names and values """ env = self.get_environment(None, log_file) env["MPI_LIB"] = "\"\"" env["FI_PSM2_DISCONNECT"] = "1" if "mpirun" in manager_cmd or "srun" in manager_cmd: env["DAOS_POOL"] = self.dfs_pool_uuid.value env["DAOS_SVCL"] = self.dfs_svcl.value env["DAOS_CONT"] = self.dfs_cont.value env["IOR_HINT__MPI__romio_daos_obj_class"] = \ self.dfs_oclass.value return env
class DfuseCommand(ExecutableCommand): """Defines a object representing a dfuse command.""" def __init__(self, namespace, command): """Create a dfuse Command object.""" super(DfuseCommand, self).__init__(namespace, command) # dfuse options self.puuid = FormattedParameter("--pool {}") self.cuuid = FormattedParameter("--container {}") self.mount_dir = FormattedParameter("--mountpoint {}") self.svcl = FormattedParameter("--svc {}", 0) self.sys_name = FormattedParameter("--sys-name {}") self.singlethreaded = FormattedParameter("--singlethreaded", False) self.foreground = FormattedParameter("--foreground", False) # Environment variable names to export when running dfuse self._env_names = [ "OFI_INTERFACE", "OFI_PORT", "CRT_PHY_ADDR_STR", "D_LOG_FILE"] def set_dfuse_params(self, pool, display=True): """Set the dfuse params for the DAOS group, pool, and container uuid. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ self.set_dfuse_pool_params(pool, display) def set_dfuse_pool_params(self, pool, display=True): """Set Dfuse params based on Daos Pool. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ self.puuid.update(pool.uuid, "puuid" if display else None) self.set_dfuse_svcl_param(pool, display) def set_dfuse_svcl_param(self, pool, display=True): """Set the dfuse svcl param from the ranks of a DAOS pool object. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ svcl = ":".join( [str(item) for item in [ int(pool.pool.svc.rl_ranks[index]) for index in range(pool.pool.svc.rl_nr)]]) self.svcl.update(svcl, "svcl" if display else None) def set_dfuse_cont_param(self, cont, display=True): """Set dfuse cont param from Container object. Args: cont (TestContainer): Daos test container object display (bool, optional): print updated params. Defaults to True. """ self.cuuid.update(cont, "cuuid" if display else None) def set_dfuse_exports(self, manager, log_file): """Set exports to issue before the dfuse command. Args: manager (DaosServerManager): server manager object to use to obtain the ofi and cart environmental variable settings from the server yaml file log_file (str): name of the log file to combine with the DAOS_TEST_LOG_DIR path with which to assign D_LOG_FILE """ env = self.get_environment(manager, log_file) self.set_environment(env)
class DmgCommandBase(YamlCommand): """Defines a base object representing a dmg command.""" def __init__(self, path, yaml_cfg=None): """Create a dmg Command object. Args: path (str): path to the dmg command yaml_cfg (DmgYamlParameters, optional): dmg config file settings. Defaults to None, in which case settings must be supplied as command-line parameters. """ super(DmgCommandBase, self).__init__("/run/dmg/*", "dmg", path, yaml_cfg) # If specified use the configuration file from the YamlParameters object default_yaml_file = None if isinstance(self.yaml, YamlParameters): default_yaml_file = self.yaml.filename self._hostlist = FormattedParameter("-l {}") self.hostfile = FormattedParameter("-f {}") self.configpath = FormattedParameter("-o {}", default_yaml_file) self.insecure = FormattedParameter("-i", False) self.debug = FormattedParameter("-d", False) self.json = FormattedParameter("-j", False) @property def hostlist(self): """Get the hostlist that was set. Returns a string list. """ if self.yaml: hosts = self.yaml.hostlist.value else: hosts = self._hostlist.value.split(",") return hosts @hostlist.setter def hostlist(self, hostlist): """Set the hostlist to be used for dmg invocation. Args: hostlist (string list): list of host addresses """ if self.yaml: if not isinstance(hostlist, list): hostlist = hostlist.split(",") self.yaml.hostlist.update(hostlist, "dmg.yaml.hostlist") else: if isinstance(hostlist, list): hostlist = ",".join(hostlist) self._hostlist.update(hostlist, "dmg._hostlist") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg sub command object based upon the sub-command.""" if self.sub_command.value == "network": self.sub_command_class = self.NetworkSubCommand() elif self.sub_command.value == "pool": self.sub_command_class = self.PoolSubCommand() elif self.sub_command.value == "storage": self.sub_command_class = self.StorageSubCommand() elif self.sub_command.value == "system": self.sub_command_class = self.SystemSubCommand() else: self.sub_command_class = None class NetworkSubCommand(CommandWithSubCommand): """Defines an object for the dmg network sub command.""" def __init__(self): """Create a dmg network subcommand object.""" super(DmgCommandBase.NetworkSubCommand, self).__init__("/run/dmg/network/*", "network") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg network sub command object.""" if self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() else: self.sub_command_class = None class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg network scan command.""" def __init__(self): """Create a dmg network scan command object.""" super(DmgCommandBase.NetworkSubCommand.ScanSubCommand, self).__init__("/run/dmg/network/scan/*", "scan") self.provider = FormattedParameter("-p {}", None) self.all = FormattedParameter("-a", False) class PoolSubCommand(CommandWithSubCommand): """Defines an object for the dmg pool sub command.""" def __init__(self): """Create a dmg pool subcommand object.""" super(DmgCommandBase.PoolSubCommand, self).__init__("/run/dmg/pool/*", "pool") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg pool sub command object.""" if self.sub_command.value == "create": self.sub_command_class = self.CreateSubCommand() elif self.sub_command.value == "delete-acl": self.sub_command_class = self.DeleteAclSubCommand() elif self.sub_command.value == "destroy": self.sub_command_class = self.DestroySubCommand() elif self.sub_command.value == "get-acl": self.sub_command_class = self.GetAclSubCommand() elif self.sub_command.value == "list": self.sub_command_class = self.ListSubCommand() elif self.sub_command.value == "overwrite-acl": self.sub_command_class = self.OverwriteAclSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "set-prop": self.sub_command_class = self.SetPropSubCommand() elif self.sub_command.value == "update-acl": self.sub_command_class = self.UpdateAclSubCommand() elif self.sub_command.value == "exclude": self.sub_command_class = self.ExcludeSubCommand() elif self.sub_command.value == "drain": self.sub_command_class = self.DrainSubCommand() elif self.sub_command.value == "reintegrate": self.sub_command_class = self.ReintegrateSubCommand() else: self.sub_command_class = None class CreateSubCommand(CommandWithParameters): """Defines an object for the dmg pool create command.""" def __init__(self): """Create a dmg pool create command object.""" super(DmgCommandBase.PoolSubCommand.CreateSubCommand, self).__init__("/run/dmg/pool/create/*", "create") self.group = FormattedParameter("--group={}", None) self.user = FormattedParameter("--user={}", None) self.acl_file = FormattedParameter("--acl-file={}", None) self.scm_size = FormattedParameter("--scm-size={}", None) self.nvme_size = FormattedParameter("--nvme-size={}", None) self.ranks = FormattedParameter("--ranks={}", None) self.nsvc = FormattedParameter("--nsvc={}", None) self.sys = FormattedParameter("--sys={}", None) class ExcludeSubCommand(CommandWithParameters): """Defines an object for the dmg pool exclude command.""" def __init__(self): """Create a dmg pool exclude command object.""" super(DmgCommandBase.PoolSubCommand.ExcludeSubCommand, self).__init__("/run/dmg/pool/exclude/*", "exclude") self.pool = FormattedParameter("--pool={}", None) self.rank = FormattedParameter("--rank={}", None) self.tgt_idx = FormattedParameter("--target-idx={}", None) class DrainSubCommand(CommandWithParameters): """Defines an object for the dmg pool drain command.""" def __init__(self): """Create a dmg pool drain command object.""" super(DmgCommandBase.PoolSubCommand.DrainSubCommand, self).__init__("/run/dmg/pool/drain/*", "drain") self.pool = FormattedParameter("--pool={}", None) self.rank = FormattedParameter("--rank={}", None) self.tgt_idx = FormattedParameter("--target-idx={}", None) class ReintegrateSubCommand(CommandWithParameters): """Defines an object for dmg pool reintegrate command.""" def __init__(self): """Create a dmg pool reintegrate command object.""" super(DmgCommandBase.PoolSubCommand.ReintegrateSubCommand, self).__init__("/run/dmg/pool/reintegrate/*", "reintegrate") self.pool = FormattedParameter("--pool={}", None) self.rank = FormattedParameter("--rank={}", None) self.tgt_idx = FormattedParameter("--target-idx={}", None) class DeleteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool delete-acl command.""" def __init__(self): """Create a dmg pool delete-acl command object.""" super(DmgCommandBase.PoolSubCommand.DeleteAclSubCommand, self).__init__("/run/dmg/pool/delete-acl/*", "delete-acl") self.pool = FormattedParameter("--pool={}", None) self.principal = FormattedParameter("-p {}", None) class DestroySubCommand(CommandWithParameters): """Defines an object for the dmg pool destroy command.""" def __init__(self): """Create a dmg pool destroy command object.""" super(DmgCommandBase.PoolSubCommand.DestroySubCommand, self).__init__("/run/dmg/pool/destroy/*", "destroy") self.pool = FormattedParameter("--pool={}", None) self.sys_name = FormattedParameter("--sys-name={}", None) self.force = FormattedParameter("--force", False) class GetAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool get-acl command.""" def __init__(self): """Create a dmg pool get-acl command object.""" super(DmgCommandBase.PoolSubCommand.GetAclSubCommand, self).__init__("/run/dmg/pool/get-acl/*", "get-acl") self.pool = FormattedParameter("--pool={}", None) class ListSubCommand(CommandWithParameters): """Defines an object for the dmg pool list command.""" def __init__(self): """Create a dmg pool list command object.""" super(DmgCommandBase.PoolSubCommand.ListSubCommand, self).__init__("/run/dmg/pool/list/*", "list") class OverwriteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool overwrite-acl command.""" def __init__(self): """Create a dmg pool overwrite-acl command object.""" super(DmgCommandBase.PoolSubCommand.OverwriteAclSubCommand, self).__init__("/run/dmg/pool/overwrite-acl/*", "overwrite-acl") self.pool = FormattedParameter("--pool={}", None) self.acl_file = FormattedParameter("-a {}", None) class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg pool query command.""" def __init__(self): """Create a dmg pool query command object.""" super(DmgCommandBase.PoolSubCommand.QuerySubCommand, self).__init__("/run/dmg/pool/query/*", "query") self.pool = FormattedParameter("--pool={}", None) class SetPropSubCommand(CommandWithParameters): """Defines an object for the dmg pool set-prop command.""" def __init__(self): """Create a dmg pool set-prop command object.""" super(DmgCommandBase.PoolSubCommand.SetPropSubCommand, self).__init__("/run/dmg/pool/set-prop/*", "set-prop") self.pool = FormattedParameter("--pool={}", None) self.name = FormattedParameter("--name={}", None) self.value = FormattedParameter("--value={}", None) class UpdateAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool update-acl command.""" def __init__(self): """Create a dmg pool update-acl command object.""" super(DmgCommandBase.PoolSubCommand.UpdateAclSubCommand, self).__init__("/run/dmg/pool/update-acl/*", "update-acl") self.pool = FormattedParameter("--pool={}", None) self.acl_file = FormattedParameter("-a {}", None) self.entry = FormattedParameter("-e {}", None) class StorageSubCommand(CommandWithSubCommand): """Defines an object for the dmg storage sub command.""" def __init__(self): """Create a dmg storage subcommand object.""" super(DmgCommandBase.StorageSubCommand, self).__init__("/run/dmg/storage/*", "storage") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg storage sub command object.""" if self.sub_command.value == "format": self.sub_command_class = self.FormatSubCommand() elif self.sub_command.value == "prepare": self.sub_command_class = self.PrepareSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() elif self.sub_command.value == "set": self.sub_command_class = self.SetSubCommand() else: self.sub_command_class = None class FormatSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage format command object.""" super(DmgCommandBase.StorageSubCommand.FormatSubCommand, self).__init__("/run/dmg/storage/format/*", "format") self.reformat = FormattedParameter("--reformat", False) class PrepareSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage prepare command object.""" super(DmgCommandBase.StorageSubCommand.PrepareSubCommand, self).__init__("/run/dmg/storage/prepare/*", "prepare") self.pci_whitelist = FormattedParameter("-w {}", None) self.hugepages = FormattedParameter("-p {}", None) self.target_user = FormattedParameter("-u {}", None) self.nvme_only = FormattedParameter("-n", False) self.scm_only = FormattedParameter("-s", False) self.reset = FormattedParameter("--reset", False) self.force = FormattedParameter("-f", False) class QuerySubCommand(CommandWithSubCommand): """Defines an object for the dmg storage query command.""" def __init__(self): """Create a dmg storage query command object.""" super(DmgCommandBase.StorageSubCommand.QuerySubCommand, self).__init__("/run/dmg/storage/query/*", "query") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg storage query sub command object.""" if self.sub_command.value == "target-health": self.sub_command_class = self.TargetHealthSubCommand() elif self.sub_command.value == "device-health": self.sub_command_class = self.DeviceHealthSubCommand() elif self.sub_command.value == "list-devices": self.sub_command_class = self.ListDevicesSubCommand() elif self.sub_command.value == "list-pools": self.sub_command_class = self.ListPoolsSubCommand() else: self.sub_command_class = None class TargetHealthSubCommand(CommandWithParameters): """Defines a dmg storage query target-health object.""" def __init__(self): """Create a dmg storage query target-health object.""" super( DmgCommandBase.StorageSubCommand.QuerySubCommand. TargetHealthSubCommand, self).__init__( "/run/dmg/storage/query/target-health/*", "target-health") self.rank = FormattedParameter("-r {}", None) self.tgtid = FormattedParameter("-t {}", None) class DeviceHealthSubCommand(CommandWithParameters): """Defines a dmg storage query device-health object.""" def __init__(self): """Create a dmg storage query device-health object.""" super( DmgCommandBase.StorageSubCommand.QuerySubCommand. DeviceHealthSubCommand, self).__init__( "/run/dmg/storage/query/device-health/*", "device-health") self.uuid = FormattedParameter("-u {}", None) class ListDevicesSubCommand(CommandWithParameters): """Defines a dmg storage query list-devices object.""" def __init__(self): """Create a dmg storage query list-devices object.""" super( DmgCommandBase.StorageSubCommand.QuerySubCommand. ListDevicesSubCommand, self).__init__("/run/dmg/storage/query/list-devices/*", "list-devices") self.rank = FormattedParameter("-r {}", None) self.uuid = FormattedParameter("-u {}", None) self.health = FormattedParameter("-b", False) class ListPoolsSubCommand(CommandWithParameters): """Defines a dmg storage query list-pools object.""" def __init__(self): """Create a dmg storage query list-pools object.""" super( DmgCommandBase.StorageSubCommand.QuerySubCommand. ListPoolsSubCommand, self).__init__("/run/dmg/storage/query/list-pools/*", "list-pools") self.rank = FormattedParameter("-r {}", None) self.uuid = FormattedParameter("-u {}", None) self.verbose = FormattedParameter("--verbose", False) class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg storage scan command.""" def __init__(self): """Create a dmg storage scan command object.""" super(DmgCommandBase.StorageSubCommand.ScanSubCommand, self).__init__("/run/dmg/storage/scan/*", "scan") self.nvme_health = FormattedParameter("--nvme-health", False) self.verbose = FormattedParameter("--verbose", False) class SetSubCommand(CommandWithSubCommand): """Defines an object for the dmg storage set command.""" def __init__(self): """Create a dmg storage set command object.""" super(DmgCommandBase.StorageSubCommand.SetSubCommand, self).__init__("/run/dmg/storage/set/*", "set") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg set sub command object.""" if self.sub_command.value == "nvme-faulty": self.sub_command_class = self.NvmeFaultySubCommand() else: self.sub_command_class = None class NvmeFaultySubCommand(CommandWithParameters): """Defines a dmg storage set nvme-faulty object.""" def __init__(self): """Create a dmg storage set nvme-faulty object.""" super( DmgCommandBase.StorageSubCommand.SetSubCommand. NvmeFaultySubCommand, self).__init__("/run/dmg/storage/query/device-state/*", "nvme-faulty") self.uuid = FormattedParameter("-u {}", None) self.force = FormattedParameter("--force", False) class SystemSubCommand(CommandWithSubCommand): """Defines an object for the dmg system sub command.""" def __init__(self): """Create a dmg system subcommand object.""" super(DmgCommandBase.SystemSubCommand, self).__init__("/run/dmg/system/*", "system") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg system sub command object.""" if self.sub_command.value == "leader-query": self.sub_command_class = self.LeaderQuerySubCommand() elif self.sub_command.value == "list-pools": self.sub_command_class = self.ListPoolsSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "start": self.sub_command_class = self.StartSubCommand() elif self.sub_command.value == "stop": self.sub_command_class = self.StopSubCommand() else: self.sub_command_class = None class LeaderQuerySubCommand(CommandWithParameters): """Defines an object for the dmg system leader-query command.""" def __init__(self): """Create a dmg system leader-query command object.""" super(DmgCommandBase.SystemSubCommand.LeaderQuerySubCommand, self).__init__("/run/dmg/system/leader-query/*", "leader-query") class ListPoolsSubCommand(CommandWithParameters): """Defines an object for the dmg system list-pools command.""" def __init__(self): """Create a dmg system list-pools command object.""" super(DmgCommandBase.SystemSubCommand.ListPoolsSubCommand, self).__init__("/run/dmg/system/list-pools/*", "list-pools") class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg system query command.""" def __init__(self): """Create a dmg system query command object.""" super(DmgCommandBase.SystemSubCommand.QuerySubCommand, self).__init__("/run/dmg/system/query/*", "query") self.ranks = FormattedParameter("--ranks={}") self.verbose = FormattedParameter("--verbose", False) class StartSubCommand(CommandWithParameters): """Defines an object for the dmg system start command.""" def __init__(self): """Create a dmg system start command object.""" super(DmgCommandBase.SystemSubCommand.StartSubCommand, self).__init__("/run/dmg/system/start/*", "start") class StopSubCommand(CommandWithParameters): """Defines an object for the dmg system stop command.""" def __init__(self): """Create a dmg system stop command object.""" super(DmgCommandBase.SystemSubCommand.StopSubCommand, self).__init__("/run/dmg/system/stop/*", "stop") self.force = FormattedParameter("--force", False) self.ranks = FormattedParameter("--ranks={}")
class DmgCommand(YamlCommand): """Defines a object representing a dmg command.""" METHOD_REGEX = { "run": r"(.*)", "network_scan": r"[-]+(?:\n|\n\r)([a-z0-9-]+)(?:\n|\n\r)[-]+|NUMA\s+" r"Socket\s+(\d+)|(ofi\+[a-z0-9;_]+)\s+([a-z0-9, ]+)", # Sample output of dmg pool list. # wolf-3:10001: connected # Pool UUID Svc Replicas # --------- ------------ # b4a27b5b-688a-4d1e-8c38-363e32eb4f29 1,2,3 # Between the first and the second group, use " +"; i.e., one or more # whitespaces. If we use "\s+", it'll pick up the second divider as # UUID since it's made up of hyphens and \s includes new line. "pool_list": r"(?:([0-9a-fA-F-]+) +([0-9,]+))", "pool_create": r"(?:UUID:|Service replicas:)\s+([A-Za-z0-9-]+)", "pool_query": r"(?:Pool\s+([0-9a-fA-F-]+),\s+ntarget=(\d+)," r"\s+disabled=(\d+),\s+leader=(\d+)," r"\s+version=(\d+)|Target\(VOS\)\s+count:\s*(\d+)|" r"(?:(?:SCM:|NVMe:)\s+Total\s+size:\s+([0-9.]+\s+[A-Z]+)" r"\s+Free:\s+([0-9.]+\s+[A-Z]+),\smin:([0-9.]+\s+[A-Z]+)" r",\s+max:([0-9.]+\s+[A-Z]+),\s+mean:([0-9.]+\s+[A-Z]+))" r"|Rebuild\s+\w+,\s+([0-9]+)\s+objs,\s+([0-9]+)" r"\s+recs)" } def __init__(self, path, yaml_cfg=None): """Create a dmg Command object. Args: path (str): path to the dmg command yaml_cfg (DmgYamlParameters, optional): dmg config file settings. Defaults to None, in which case settings must be supplied as command-line parameters. """ super(DmgCommand, self).__init__("/run/dmg/*", "dmg", path, yaml_cfg) # If specified use the configuration file from the YamlParameters object default_yaml_file = None if isinstance(self.yaml, YamlParameters): default_yaml_file = self.yaml.filename self._hostlist = FormattedParameter("-l {}") self.hostfile = FormattedParameter("-f {}") self.configpath = FormattedParameter("-o {}", default_yaml_file) self.insecure = FormattedParameter("-i", False) self.debug = FormattedParameter("-d", False) self.json = FormattedParameter("-j", False) @property def hostlist(self): """Get the hostlist that was set. Returns a string list. """ if self.yaml: return self.yaml.hostlist.value else: return self._hostlist.value.split(",") @hostlist.setter def hostlist(self, hostlist): """Set the hostlist to be used for dmg invocation. Args: hostlist (string list): list of host addresses """ if self.yaml: if not isinstance(hostlist, list): hostlist = hostlist.split(",") self.yaml.hostlist.update(hostlist, "dmg.yaml.hostlist") else: if isinstance(hostlist, list): hostlist = ",".join(hostlist) self._hostlist.update(hostlist, "dmg._hostlist") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg sub command object based upon the sub-command.""" if self.sub_command.value == "network": self.sub_command_class = self.NetworkSubCommand() elif self.sub_command.value == "pool": self.sub_command_class = self.PoolSubCommand() elif self.sub_command.value == "storage": self.sub_command_class = self.StorageSubCommand() elif self.sub_command.value == "system": self.sub_command_class = self.SystemSubCommand() else: self.sub_command_class = None class NetworkSubCommand(CommandWithSubCommand): """Defines an object for the dmg network sub command.""" def __init__(self): """Create a dmg network subcommand object.""" super(DmgCommand.NetworkSubCommand, self).__init__("/run/dmg/network/*", "network") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg network sub command object.""" if self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() else: self.sub_command_class = None class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg network scan command.""" def __init__(self): """Create a dmg network scan command object.""" super(DmgCommand.NetworkSubCommand.ScanSubCommand, self).__init__("/run/dmg/network/scan/*", "scan") self.provider = FormattedParameter("-p {}", None) self.all = FormattedParameter("-a", False) class PoolSubCommand(CommandWithSubCommand): """Defines an object for the dmg pool sub command.""" def __init__(self): """Create a dmg pool subcommand object.""" super(DmgCommand.PoolSubCommand, self).__init__("/run/dmg/pool/*", "pool") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg pool sub command object.""" if self.sub_command.value == "create": self.sub_command_class = self.CreateSubCommand() elif self.sub_command.value == "delete-acl": self.sub_command_class = self.DeleteAclSubCommand() elif self.sub_command.value == "destroy": self.sub_command_class = self.DestroySubCommand() elif self.sub_command.value == "get-acl": self.sub_command_class = self.GetAclSubCommand() elif self.sub_command.value == "list": self.sub_command_class = self.ListSubCommand() elif self.sub_command.value == "overwrite-acl": self.sub_command_class = self.OverwriteAclSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "set-prop": self.sub_command_class = self.SetPropSubCommand() elif self.sub_command.value == "update-acl": self.sub_command_class = self.UpdateAclSubCommand() elif self.sub_command.value == "exclude": self.sub_command_class = self.ExcludeSubCommand() elif self.sub_command.value == "reintegrate": self.sub_command_class = self.ReintegrateSubCommand() else: self.sub_command_class = None class CreateSubCommand(CommandWithParameters): """Defines an object for the dmg pool create command.""" def __init__(self): """Create a dmg pool create command object.""" super(DmgCommand.PoolSubCommand.CreateSubCommand, self).__init__("/run/dmg/pool/create/*", "create") self.group = FormattedParameter("--group={}", None) self.user = FormattedParameter("--user={}", None) self.acl_file = FormattedParameter("--acl-file={}", None) self.scm_size = FormattedParameter("--scm-size={}", None) self.nvme_size = FormattedParameter("--nvme-size={}", None) self.ranks = FormattedParameter("--ranks={}", None) self.nsvc = FormattedParameter("--nsvc={}", None) self.sys = FormattedParameter("--sys={}", None) class ExcludeSubCommand(CommandWithParameters): """Defines an object for the dmg pool exclude command.""" def __init__(self): """Create a dmg pool exclude command object.""" super(DmgCommand.PoolSubCommand.ExcludeSubCommand, self).__init__("/run/dmg/pool/exclude/*", "exclude") self.pool = FormattedParameter("--pool={}", None) self.rank = FormattedParameter("--rank={}", None) self.tgt_idx = FormattedParameter("--target-idx={}", None) class ReintegrateSubCommand(CommandWithParameters): """Defines an object for dmg pool reintegrate command.""" def __init__(self): """Create a dmg pool reintegrate command object.""" super(DmgCommand.PoolSubCommand.ReintegrateSubCommand, self).__init__("/run/dmg/pool/reintegrate/*", "reintegrate") self.pool = FormattedParameter("--pool={}", None) self.rank = FormattedParameter("--rank={}", None) self.tgt_idx = FormattedParameter("--target-idx={}", None) class DeleteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool delete-acl command.""" def __init__(self): """Create a dmg pool delete-acl command object.""" super(DmgCommand.PoolSubCommand.DeleteAclSubCommand, self).__init__("/run/dmg/pool/delete-acl/*", "delete-acl") self.pool = FormattedParameter("--pool={}", None) self.principal = FormattedParameter("-p {}", None) class DestroySubCommand(CommandWithParameters): """Defines an object for the dmg pool destroy command.""" def __init__(self): """Create a dmg pool destroy command object.""" super(DmgCommand.PoolSubCommand.DestroySubCommand, self).__init__("/run/dmg/pool/destroy/*", "destroy") self.pool = FormattedParameter("--pool={}", None) self.sys_name = FormattedParameter("--sys-name={}", None) self.force = FormattedParameter("--force", False) class GetAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool get-acl command.""" def __init__(self): """Create a dmg pool get-acl command object.""" super(DmgCommand.PoolSubCommand.GetAclSubCommand, self).__init__("/run/dmg/pool/get-acl/*", "get-acl") self.pool = FormattedParameter("--pool={}", None) class ListSubCommand(CommandWithParameters): """Defines an object for the dmg pool list command.""" def __init__(self): """Create a dmg pool list command object.""" super(DmgCommand.PoolSubCommand.ListSubCommand, self).__init__("/run/dmg/pool/list/*", "list") class OverwriteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool overwrite-acl command.""" def __init__(self): """Create a dmg pool overwrite-acl command object.""" super(DmgCommand.PoolSubCommand.OverwriteAclSubCommand, self).__init__("/run/dmg/pool/overwrite-acl/*", "overwrite-acl") self.pool = FormattedParameter("--pool={}", None) self.acl_file = FormattedParameter("-a {}", None) class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg pool query command.""" def __init__(self): """Create a dmg pool query command object.""" super(DmgCommand.PoolSubCommand.QuerySubCommand, self).__init__("/run/dmg/pool/query/*", "query") self.pool = FormattedParameter("--pool={}", None) class SetPropSubCommand(CommandWithParameters): """Defines an object for the dmg pool set-prop command.""" def __init__(self): """Create a dmg pool set-prop command object.""" super(DmgCommand.PoolSubCommand.SetPropSubCommand, self).__init__("/run/dmg/pool/set-prop/*", "set-prop") self.pool = FormattedParameter("--pool={}", None) self.name = FormattedParameter("--name={}", None) self.value = FormattedParameter("--value={}", None) class UpdateAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool update-acl command.""" def __init__(self): """Create a dmg pool update-acl command object.""" super(DmgCommand.PoolSubCommand.UpdateAclSubCommand, self).__init__("/run/dmg/pool/update-acl/*", "update-acl") self.pool = FormattedParameter("--pool={}", None) self.acl_file = FormattedParameter("-a {}", None) self.entry = FormattedParameter("-e {}", None) class StorageSubCommand(CommandWithSubCommand): """Defines an object for the dmg storage sub command.""" def __init__(self): """Create a dmg storage subcommand object.""" super(DmgCommand.StorageSubCommand, self).__init__("/run/dmg/storage/*", "storage") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg storage sub command object.""" if self.sub_command.value == "format": self.sub_command_class = self.FormatSubCommand() elif self.sub_command.value == "prepare": self.sub_command_class = self.PrepareSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() elif self.sub_command.value == "set": self.sub_command_class = self.SetSubCommand() else: self.sub_command_class = None class FormatSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage format command object.""" super(DmgCommand.StorageSubCommand.FormatSubCommand, self).__init__("/run/dmg/storage/format/*", "format") self.reformat = FormattedParameter("--reformat", False) class PrepareSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage prepare command object.""" super(DmgCommand.StorageSubCommand.PrepareSubCommand, self).__init__("/run/dmg/storage/prepare/*", "prepare") self.pci_whitelist = FormattedParameter("-w {}", None) self.hugepages = FormattedParameter("-p {}", None) self.target_user = FormattedParameter("-u {}", None) self.nvme_only = FormattedParameter("-n", False) self.scm_only = FormattedParameter("-s", False) self.reset = FormattedParameter("--reset", False) self.force = FormattedParameter("-f", False) class QuerySubCommand(CommandWithSubCommand): """Defines an object for the dmg query format command.""" def __init__(self): """Create a dmg storage query command object.""" super(DmgCommand.StorageSubCommand.QuerySubCommand, self).__init__("/run/dmg/storage/query/*", "query") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg pool sub command object.""" if self.sub_command.value == "blobstore-health": self.sub_command_class = self.BlobstoreHealthSubCommand() elif self.sub_command.value == "smd": self.sub_command_class = self.SmdSubCommand() else: self.sub_command_class = None class BlobstoreHealthSubCommand(CommandWithParameters): """Defines a dmg storage query blobstore-health object.""" def __init__(self): """Create a dmg storage query blobstore-health object.""" super( DmgCommand.StorageSubCommand.QuerySubCommand. BlobstoreHealthSubCommand, self).__init__( "/run/dmg/storage/query/blobstore-health/*", "blobstore-health") self.devuuid = FormattedParameter("-u {}", None) self.tgtid = FormattedParameter("-t {}", None) class SmdSubCommand(CommandWithParameters): """Defines a dmg storage query smd object.""" def __init__(self): """Create a dmg storage query smd object.""" super( DmgCommand.StorageSubCommand.QuerySubCommand. SmdSubCommand, self).__init__("/run/dmg/storage/query/smd/*", "smd") self.devices = FormattedParameter("-d", False) self.pools = FormattedParameter("-p", False) class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg storage scan command.""" def __init__(self): """Create a dmg storage scan command object.""" super(DmgCommand.StorageSubCommand.ScanSubCommand, self).__init__("/run/dmg/storage/scan/*", "scan") self.summary = FormattedParameter("-m", False) class SetSubCommand(CommandWithParameters): """Defines an object for the dmg storage set command.""" def __init__(self): """Create a dmg storage set command object.""" super(DmgCommand.StorageSubCommand.SetSubCommand, self).__init__("/run/dmg/storage/set/*", "set") self.nvme_faulty = FormattedParameter("nvme-faulty", False) class SystemSubCommand(CommandWithSubCommand): """Defines an object for the dmg system sub command.""" def __init__(self): """Create a dmg system subcommand object.""" super(DmgCommand.SystemSubCommand, self).__init__("/run/dmg/system/*", "system") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg system sub command object.""" if self.sub_command.value == "leader-query": self.sub_command_class = self.LeaderQuerySubCommand() elif self.sub_command.value == "list-pools": self.sub_command_class = self.ListPoolsSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "start": self.sub_command_class = self.StartSubCommand() elif self.sub_command.value == "stop": self.sub_command_class = self.StopSubCommand() else: self.sub_command_class = None class LeaderQuerySubCommand(CommandWithParameters): """Defines an object for the dmg system leader-query command.""" def __init__(self): """Create a dmg system leader-query command object.""" super(DmgCommand.SystemSubCommand.LeaderQuerySubCommand, self).__init__("/run/dmg/system/leader-query/*", "leader-query") class ListPoolsSubCommand(CommandWithParameters): """Defines an object for the dmg system list-pools command.""" def __init__(self): """Create a dmg system list-pools command object.""" super(DmgCommand.SystemSubCommand.ListPoolsSubCommand, self).__init__("/run/dmg/system/list-pools/*", "list-pools") class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg system query command.""" def __init__(self): """Create a dmg system query command object.""" super(DmgCommand.SystemSubCommand.QuerySubCommand, self).__init__("/run/dmg/system/query/*", "query") self.rank = FormattedParameter("--rank={}") self.verbose = FormattedParameter("--verbose", False) class StartSubCommand(CommandWithParameters): """Defines an object for the dmg system start command.""" def __init__(self): """Create a dmg system start command object.""" super(DmgCommand.SystemSubCommand.StartSubCommand, self).__init__("/run/dmg/system/start/*", "start") class StopSubCommand(CommandWithParameters): """Defines an object for the dmg system stop command.""" def __init__(self): """Create a dmg system stop command object.""" super(DmgCommand.SystemSubCommand.StopSubCommand, self).__init__("/run/dmg/system/stop/*", "stop") self.force = FormattedParameter("--force", False) def network_scan(self, provider=None, all_devs=False): """Get the result of the dmg network scan command. Args: provider (str): name of network provider tied to the device all_devs (bool, optional): Show all devs info. Defaults to False. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage scan command fails. """ self.set_sub_command("network") self.sub_command_class.set_sub_command("scan") self.sub_command_class.sub_command_class.provider.value = provider self.sub_command_class.sub_command_class.all.value = all_devs return self._get_result() def storage_scan(self): """Get the result of the dmg storage scan command. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage scan command fails. """ self.set_sub_command("storage") self.sub_command_class.set_sub_command("scan") return self._get_result() def storage_format(self, reformat=False): """Get the result of the dmg storage format command. Args: reformat (bool): always reformat storage, could be destructive. This will create control-plane related metadata i.e. superblock file and reformat if the storage media is available and formattable. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage format command fails. """ self.set_sub_command("storage") self.sub_command_class.set_sub_command("format") self.sub_command_class.sub_command_class.reformat.value = reformat return self._get_result() def storage_prepare(self, user=None, hugepages="4096", nvme=False, scm=False, reset=False, force=True): """Get the result of the dmg storage format command. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg storage prepare command fails. """ self.set_sub_command("storage") self.sub_command_class.set_sub_command("prepare") self.sub_command_class.sub_command_class.nvme_only.value = nvme self.sub_command_class.sub_command_class.scm_only.value = scm self.sub_command_class.sub_command_class.target_user.value = \ getuser() if user is None else user self.sub_command_class.sub_command_class.hugepages.value = hugepages self.sub_command_class.sub_command_class.reset.value = reset self.sub_command_class.sub_command_class.force.value = force return self._get_result() def pool_create(self, scm_size, uid=None, gid=None, nvme_size=None, target_list=None, svcn=None, group=None, acl_file=None): """Create a pool with the dmg command. The uid and gid method arguments can be specified as either an integer or a string. If an integer value is specified it will be converted into the corresponding user/group name string. Args: scm_size (int): SCM pool size to create. uid (object, optional): User ID with privileges. Defaults to None. gid (object, optional): Group ID with privileges. Defaults to None. nvme_size (str, optional): NVMe size. Defaults to None. target_list (list, optional): a list of storage server unique identifiers (ranks) for the DAOS pool svcn (str, optional): Number of pool service replicas. Defaults to None, in which case 1 is used by the dmg binary in default. group (str, optional): DAOS system group name in which to create the pool. Defaults to None, in which case "daos_server" is used by default. acl_file (str, optional): ACL file. Defaults to None. Returns: CmdResult: an avocado CmdResult object containing the dmg command information, e.g. exit status, stdout, stderr, etc. Raises: CommandFailure: if the dmg pool create command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("create") self.sub_command_class.sub_command_class.user.value = \ getpwuid(uid).pw_name if isinstance(uid, int) else uid self.sub_command_class.sub_command_class.group.value = \ getgrgid(gid).gr_name if isinstance(gid, int) else gid self.sub_command_class.sub_command_class.scm_size.value = scm_size self.sub_command_class.sub_command_class.nvme_size.value = nvme_size if target_list is not None: self.sub_command_class.sub_command_class.ranks.value = ",".join( [str(target) for target in target_list]) self.sub_command_class.sub_command_class.nsvc.value = svcn self.sub_command_class.sub_command_class.sys.value = group self.sub_command_class.sub_command_class.acl_file.value = acl_file return self._get_result() def pool_query(self, pool): """Query a pool with the dmg command. Args: uuid (str): Pool UUID to query. Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool query command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("query") self.sub_command_class.sub_command_class.pool.value = pool return self._get_result() def pool_destroy(self, pool, force=True): """Destroy a pool with the dmg command. Args: pool (str): Pool UUID to destroy. force (bool, optional): Force removal of pool. Defaults to True. Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool destroy command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("destroy") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.force.value = force return self._get_result() def pool_get_acl(self, pool): """Get the ACL for a given pool. Args: pool (str): Pool for which to get the ACL. Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool get-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("get-acl") self.sub_command_class.sub_command_class.pool.value = pool return self._get_result() def pool_update_acl(self, pool, acl_file, entry): """Update the acl for a given pool. Args: pool (str): Pool for which to update the ACL. acl_file (str): ACL file to update entry (str): entry to be updated Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool update-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("update-acl") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.acl_file.value = acl_file self.sub_command_class.sub_command_class.entry.value = entry return self._get_result() def pool_overwrite_acl(self, pool, acl_file): """Overwrite the acl for a given pool. Args: pool (str): Pool for which to overwrite the ACL. acl_file (str): ACL file to update Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool overwrite-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("overwrite-acl") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.acl_file.value = acl_file return self._get_result() def pool_delete_acl(self, pool, principal): """Delete the acl for a given pool. Args: pool (str): Pool for which to delete the ACL. principal (str): principal to be deleted Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool delete-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("delete-acl") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.principal.value = principal return self._get_result() def pool_list(self): """List pools. Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool delete-acl command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("list") return self._get_result() def pool_set_prop(self, pool, name, value): """Set property for a given Pool. Args: pool (str): Pool uuid for which property is supposed to be set. name (str): Property name to be set value (str): Property value to be set Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool set-prop command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("set-prop") self.sub_command_class.sub_command_class.pool.value = pool self.sub_command_class.sub_command_class.name.value = name self.sub_command_class.sub_command_class.value.value = value return self._get_result() def pool_exclude(self, pool_uuid, rank, tgt_idx=None): """Exclude a daos_server from the pool Args: pool (str): Pool uuid. rank (int): Rank of the daos_server to exclude tgt_idx (int): target to be excluded from the pool Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool exclude command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("exclude") self.sub_command_class.sub_command_class.pool.value = pool_uuid self.sub_command_class.sub_command_class.rank.value = rank self.sub_command_class.sub_command_class.tgt_idx.value = tgt_idx return self._get_result() def pool_reintegrate(self, pool_uuid, rank, tgt_idx=None): """Reintegrate a daos_server to the pool Args: pool (str): Pool uuid. rank (int): Rank of the daos_server to reintegrate tgt_idx (int): target to be reintegrated to the pool Returns: CmdResult: Object that contains exit status, stdout, and other information. Raises: CommandFailure: if the dmg pool reintegrate command fails. """ self.set_sub_command("pool") self.sub_command_class.set_sub_command("reintegrate") self.sub_command_class.sub_command_class.pool.value = pool_uuid self.sub_command_class.sub_command_class.rank.value = rank self.sub_command_class.sub_command_class.tgt_idx.value = tgt_idx return self._get_result()
class Mpirun(JobManager): """A class for the mpirun job manager command.""" def __init__(self, job, subprocess=False, mpi_type="openmpi"): """Create a Mpirun object. Args: job (ExecutableCommand): command object to manage. subprocess (bool, optional): whether the command is run as a subprocess. Defaults to False. """ if not load_mpi(mpi_type): raise MPILoadError(mpi_type) path = os.path.dirname(find_executable("mpirun")) super().__init__("/run/mpirun", "mpirun", job, path, subprocess) mca_default = None if mpi_type == "openmpi": # Default mca values to avoid queue pair errors w/ OpenMPI mca_default = { "btl_openib_warn_default_gid_prefix": "0", "btl": "tcp,self", "oob": "tcp", "pml": "ob1", "btl_tcp_if_include": "eth0", } self.hostfile = FormattedParameter("-hostfile {}", None) self.processes = FormattedParameter("-np {}", 1) self.ppn = FormattedParameter("-ppn {}", None) self.envlist = FormattedParameter("-envlist {}", None) if mpi_type == "openmpi": self.genv = FormattedParameter("-x {}", None) else: self.genv = FormattedParameter("-genv {}", None) self.mca = FormattedParameter("--mca {}", mca_default) self.working_dir = FormattedParameter("-wdir {}", None) self.tmpdir_base = FormattedParameter("--mca orte_tmpdir_base {}", None) self.mpi_type = mpi_type def assign_hosts(self, hosts, path=None, slots=None): """Assign the hosts to use with the command (-f). Args: hosts (list): list of hosts to specify in the hostfile path (str, optional): hostfile path. Defaults to None. slots (int, optional): number of slots per host to specify in the hostfile. Defaults to None. """ self._hosts = hosts kwargs = {"hostlist": self._hosts, "slots": slots} if path is not None: kwargs["path"] = path self.hostfile.value = write_host_file(**kwargs) def assign_processes(self, processes): """Assign the number of processes per node (-np). Args: processes (int): number of processes per node """ self.processes.update(processes, "mpirun.np") def assign_environment(self, env_vars, append=False): """Assign or add environment variables to the command. Args: env_vars (EnvironmentVariables): the environment variables to use assign or add to the command append (bool): whether to assign (False) or append (True) the specified environment variables """ # Pass the environment variables via the process.run method env argument if append and self.env is not None: # Update the existing dictionary with the new values self.genv.update(env_vars.get_list()) else: # Overwrite/create the dictionary of environment variables self.genv.update((EnvironmentVariables(env_vars)).get_list()) def assign_environment_default(self, env_vars): """Assign the default environment variables for the command. Args: env_vars (EnvironmentVariables): the environment variables to assign as the default """ self.genv.update_default(env_vars.get_list()) def run(self): """Run the mpirun command. Raises: CommandFailure: if there is an error running the command """ if not load_mpi(self.mpi_type): raise MPILoadError(self.mpi_type) return super().run()
class IorCommand(ExecutableCommand): """Defines a object for executing an IOR command. Example: >>> # Typical use inside of a DAOS avocado test method. >>> ior_cmd = IorCommand() >>> ior_cmd.get_params(self) >>> ior_cmd.set_daos_params(self.server_group, self.pool) >>> mpirun = Mpirun() >>> server_manager = self.server_manager[0] >>> env = self.ior_cmd.get_environment(server_manager, self.client_log) >>> processes = len(self.hostlist_clients) >>> mpirun.setup_command(env, self.hostfile_clients, processes) >>> mpirun.run() """ def __init__(self): """Create an IorCommand object.""" super(IorCommand, self).__init__("/run/ior/*", "ior") # Flags self.flags = FormattedParameter("{}") # Optional arguments # -a=POSIX API for I/O [POSIX|DUMMY|MPIIO|MMAP|DAOS|DFS] # -b=1048576 blockSize -- contiguous bytes to write per task # -d=0 interTestDelay -- delay between reps in seconds # -f=STRING scriptFile -- test script name # -G=0 setTimeStampSignature -- time stamp signature # -i=1 repetitions -- number of repetitions of test # -j=0 outlierThreshold -- warn on outlier N sec from mean # -J=1 setAlignment -- HDF5 alignment in bytes # -l=STRING datapacket type-- type of packet created # -M=STRING memoryPerNode -- hog memory on the node # -N=0 numTasks -- num of participating tasks in the test # -o=testFile testFile -- full name for test # -O=STRING string of IOR directives # -Q=1 taskPerNodeOffset for read tests # -s=1 segmentCount -- number of segments # -t=262144 transferSize -- size of transfer in bytes # -T=0 maxTimeDuration -- max time in minutes executing # repeated test; it aborts only between iterations # and not within a test! self.api = FormattedParameter("-a {}", "DAOS") self.block_size = FormattedParameter("-b {}") self.test_delay = FormattedParameter("-d {}") self.script = FormattedParameter("-f {}") self.signatute = FormattedParameter("-G {}") self.repetitions = FormattedParameter("-i {}") self.outlier_threshold = FormattedParameter("-j {}") self.alignment = FormattedParameter("-J {}") self.data_packet_type = FormattedParameter("-l {}") self.memory_per_node = FormattedParameter("-M {}") self.num_tasks = FormattedParameter("-N {}") self.test_file = FormattedParameter("-o {}") self.directives = FormattedParameter("-O {}") self.task_offset = FormattedParameter("-Q {}") self.segment_count = FormattedParameter("-s {}") self.transfer_size = FormattedParameter("-t {}") self.max_duration = FormattedParameter("-T {}") # Module DAOS # Required arguments # --daos.pool=STRING pool uuid # --daos.svcl=STRING pool SVCL # --daos.cont=STRING container uuid # Flags # --daos.destroy Destroy Container # Optional arguments # --daos.group=STRING server group # --daos.chunk_size=1048576 chunk size # --daos.oclass=STRING object class self.daos_pool = FormattedParameter("--daos.pool {}") self.daos_svcl = FormattedParameter("--daos.svcl {}") self.daos_cont = FormattedParameter("--daos.cont {}") self.daos_destroy = FormattedParameter("--daos.destroy", True) self.daos_group = FormattedParameter("--daos.group {}") self.daos_chunk = FormattedParameter("--daos.chunk_size {}", 1048576) self.daos_oclass = FormattedParameter("--daos.oclass {}", "SX") # Module DFS # Required arguments # --dfs.pool=STRING pool uuid # --dfs.svcl=STRING pool SVCL # --dfs.cont=STRING container uuid # Flags # --daos.destroy Destroy Container # Optional arguments # --dfs.group=STRING server group # --dfs.chunk_size=1048576 chunk size # --dfs.oclass=STRING object class # --dfs.prefix=STRING mount prefix self.dfs_pool = FormattedParameter("--dfs.pool {}") self.dfs_svcl = FormattedParameter("--dfs.svcl {}") self.dfs_cont = FormattedParameter("--dfs.cont {}") self.dfs_destroy = FormattedParameter("--dfs.destroy", True) self.dfs_group = FormattedParameter("--dfs.group {}") self.dfs_chunk = FormattedParameter("--dfs.chunk_size {}", 1048576) self.dfs_oclass = FormattedParameter("--dfs.oclass {}", "SX") self.dfs_prefix = FormattedParameter("--dfs.prefix {}") # A list of environment variable names to set and export with ior self._env_names = ["D_LOG_FILE"] def get_param_names(self): """Get a sorted list of the defined IorCommand parameters.""" # Sort the IOR parameter names to generate consistent ior commands all_param_names = super(IorCommand, self).get_param_names() # List all of the common ior params first followed by any daos-specific # and dfs-specific params (except when using MPIIO). param_names = [ name for name in all_param_names if ("daos" not in name) and ("dfs" not in name) ] if self.api.value == "DAOS": param_names.extend( [name for name in all_param_names if "daos" in name]) elif self.api.value == "DFS": param_names.extend( [name for name in all_param_names if "dfs" in name]) return param_names def set_daos_params(self, group, pool, cont_uuid=None, display=True): """Set the IOR parameters for the DAOS group, pool, and container uuid. Args: group (str): DAOS server group name pool (TestPool): DAOS test pool object cont_uuid (str, optional): the container uuid. If not specified one is generated. Defaults to None. display (bool, optional): print updated params. Defaults to True. """ self.set_daos_pool_params(pool, display) if self.api.value in ["DAOS", "MPIIO"]: self.daos_group.update(group, "daos_group" if display else None) self.daos_cont.update(cont_uuid if cont_uuid else uuid.uuid4(), "daos_cont" if display else None) else: self.dfs_group.update(group, "daos_group" if display else None) self.dfs_cont.update(cont_uuid if cont_uuid else uuid.uuid4(), "daos_cont" if display else None) def set_daos_pool_params(self, pool, display=True): """Set the IOR parameters that are based on a DAOS pool. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ if self.api.value in ["DAOS", "MPIIO"]: self.daos_pool.update(pool.pool.get_uuid_str(), "daos_pool" if display else None) else: self.dfs_pool.update(pool.pool.get_uuid_str(), "dfs_pool" if display else None) self.set_daos_svcl_param(pool, display) def set_daos_svcl_param(self, pool, display=True): """Set the IOR daos_svcl param from the ranks of a DAOS pool object. Args: pool (TestPool): DAOS test pool object display (bool, optional): print updated params. Defaults to True. """ svcl = ":".join([ str(item) for item in [ int(pool.pool.svc.rl_ranks[index]) for index in range(pool.pool.svc.rl_nr) ] ]) if self.api.value in ["DAOS", "MPIIO"]: self.daos_svcl.update(svcl, "daos_svcl" if display else None) else: self.dfs_svcl.update(svcl, "dfs_svcl" if display else None) def get_aggregate_total(self, processes): """Get the total bytes expected to be written by ior. Args: processes (int): number of processes running the ior command Returns: int: total number of bytes written Raises: CommandFailure: if there is an error obtaining the aggregate total """ power = {"k": 1, "m": 2, "g": 3, "t": 4} total = processes for name in ("block_size", "segment_count"): item = getattr(self, name).value if item: sub_item = re.split(r"([^\d])", str(item)) if sub_item > 0: total *= int(sub_item[0]) if len(sub_item) > 1: key = sub_item[1].lower() if key in power: total *= 1024**power[key] else: raise CommandFailure( "Error obtaining the IOR aggregate total from " "the {} - bad key: value: {}, split: {}, " "key: {}".format(name, item, sub_item, key)) else: raise CommandFailure( "Error obtaining the IOR aggregate total from the {}: " "value: {}, split: {}".format(name, item, sub_item)) # Account for any replicas, except for the ones with no replication # i.e all object classes starting with "S". Eg: S1,S2,...,SX. if not self.daos_oclass.value.startswith("S"): try: # Extract the replica quantity from the object class string replica_qty = int( re.findall(r"\d+", self.daos_oclass.value)[0]) except (TypeError, IndexError): # If the daos object class is undefined (TypeError) or it does # not contain any numbers (IndexError) then there is only one # replica. replica_qty = 1 finally: total *= replica_qty return total def get_default_env(self, manager_cmd, log_file=None): """Get the default enviroment settings for running IOR. Args: manager_cmd (str): job manager command log_file (str, optional): log file. Defaults to None. Returns: EnvironmentVariables: a dictionary of environment names and values """ env = self.get_environment(None, log_file) env["MPI_LIB"] = "\"\"" env["FI_PSM2_DISCONNECT"] = "1" if "mpirun" in manager_cmd or "srun" in manager_cmd: env["DAOS_POOL"] = self.daos_pool.value env["DAOS_SVCL"] = self.daos_svcl.value env["DAOS_CONT"] = self.daos_cont.value env["IOR_HINT__MPI__romio_daos_obj_class"] = self.daos_oclass.value return env @staticmethod def get_ior_metrics(cmdresult): """Get the ior command read and write metrics. Parse the CmdResult (output of the test) and look for the ior stdout and get the read and write metrics. Args: cmdresult (CmdResult): output of job manager Returns: metrics (tuple) : list of write and read metrics from ior run """ ior_metric_summary = "Summary of all tests:" messages = cmdresult.stdout.splitlines() # Get the index whre the summary starts and add one to # get to the header. idx = messages.index(ior_metric_summary) # idx + 1 is header. # idx +2 and idx + 3 will give the write and read metrics. write_metrics = (" ".join(messages[idx + 2].split())).split() read_metrics = (" ".join(messages[idx + 3].split())).split() return (write_metrics, read_metrics) @staticmethod def log_metrics(logger, message, metrics): """Log the ior metrics. Args: logger (log): logger object handle message (str) : Message to print before logging metrics metric (lst) : IOR write and read metrics """ logger.info("\n") logger.info(message) for m in metrics: logger.info(m) logger.info("\n")
class DataMoverCommand(ExecutableCommand): """Defines a object representing a datamover command.""" def __init__(self, namespace, command): """Create a datamover Command object.""" super(DataMoverCommand, self).__init__(namespace, command) # datamover options # IO buffer size in bytes (default 1MB) self.blocksize = FormattedParameter("--blocksize {}") # DAOS source pool self.daos_src_pool = FormattedParameter("--daos-src-pool {}") # DAOS destination pool self.daos_dst_pool = FormattedParameter("--daos-dst-pool {}") # DAOS source container self.daos_src_cont = FormattedParameter("--daos-src-cont {}") # DAOS destination container self.daos_dst_cont = FormattedParameter("--daos-dst-cont {}") # DAOS prefix for unified namespace path self.daos_prefix = FormattedParameter("--daos-prefix {}") # read source list from file self.input_file = FormattedParameter("--input {}") # work size per task in bytes (default 1MB) self.chunksize = FormattedParameter("--chunksize {}") # preserve permissions, ownership, timestamps, extended attributes self.preserve = FormattedParameter("--preserve", False) # use synchronous read/write calls (O_DIRECT) self.direct = FormattedParameter("--direct", False) # create sparse files when possible self.sparse = FormattedParameter("--sparse", False) # print progress every N seconds self.progress = FormattedParameter("--progress {}") # verbose output self.verbose = FormattedParameter("--verbose", False) # quiet output self.quiet = FormattedParameter("--quiet", False) # print help/usage self.print_usage = FormattedParameter("--help", False) # source path self.src_path = BasicParameter(None) # destination path self.dest_path = BasicParameter(None) def get_param_names(self): """Overriding the original get_param_names""" param_names = super(DataMoverCommand, self).get_param_names() # move key=dest_path to the end param_names.sort(key='dest_path'.__eq__) return param_names def set_datamover_params(self, src_pool=None, dst_pool=None, src_cont=None, dst_cont=None, display=True): """Set the datamover params for the DAOS group, pool, and cont uuid. Args: src_pool(TestPool): source pool object dst_pool(TestPool): destination pool object src_cont(TestContainer): source container object dst_cont(TestContainer): destination container object display (bool, optional): print updated params. Defaults to True. """ # set the obtained values if src_pool: self.daos_src_pool.update(src_pool.uuid, "daos_src_pool" if display else None) if dst_pool: self.daos_dst_pool.update(dst_pool.uuid, "daos_dst_pool" if display else None) if src_cont: self.daos_src_cont.update(src_cont.uuid, "daos_src_cont" if display else None) if dst_cont: self.daos_dst_cont.update(dst_cont.uuid, "daos_dst_cont" if display else None)
class DmgCommandBase(YamlCommand): """Defines a base object representing a dmg command.""" METHOD_REGEX = { "run": r"(.*)", "network_scan": r"[-]+(?:\n|\n\r)([a-z0-9-]+)(?:\n|\n\r)[-]+|NUMA\s+" r"Socket\s+(\d+)|(ofi\+[a-z0-9;_]+)\s+([a-z0-9, ]+)", "pool_list": r"(?:([0-9a-fA-F-]+) +([0-9,]+))", "pool_create": r"(?:UUID:|Service replicas:)\s+([A-Za-z0-9-]+)", "pool_query": r"(?:Pool\s+([0-9a-fA-F-]+),\s+ntarget=(\d+),\s+disabled=(\d+)," r"\s+leader=(\d+),\s+version=(\d+)|Target\(VOS\)\s+count:" r"\s*(\d+)|(?:(?:SCM:|NVMe:)\s+Total\s+size:\s+([0-9.]+\s+[A-Z]+)" r"\s+Free:\s+([0-9.]+\s+[A-Z]+),\smin:([0-9.]+\s+[A-Z]+)," r"\s+max:([0-9.]+\s+[A-Z]+),\s+mean:([0-9.]+\s+[A-Z]+))" r"|Rebuild\s+\w+,\s+([0-9]+)\s+objs,\s+([0-9]+)\s+recs)", "storage_query_list_pools": r"[-]+\s+([a-z0-9-]+)\s+[-]+|(?:UUID:([a-z0-9-]+)\s+Rank:([0-9]+)" r"\s+Targets:\[([0-9 ]+)\])(?:\s+Blobs:\[([0-9 ]+)\]\s+?$)", "storage_query_list_devices": r"[-]+\s+([a-z0-9-]+)\s+[-]+\s+.*\s+|(?:UUID:([a-z0-9-]+)\s+" r"Targets:\[([0-9 ]+)\]\s+Rank:([0-9]+)\s+State:([A-Z]+))", "storage_query_device_health": r"[-]+\s+([a-z0-9-]+)\s+[-]+\s+.*\s+UUID:([a-z0-9-]+)\s+Targets:" r"\[([0-9 ]+)\]\s+Rank:([0-9]+)\s+State:(\w+)\s+.*\s+|(?:Temp.*|" r"Cont.*Busy Time|Pow.*Cycles|Pow.*Duration|Unsafe.*|Media.*|" r"Read.*|Write.*|Unmap.*|Checksum.*|Err.*Entries|Avail.*|" r"Dev.*Reli.*|Vola.*):\s*([A-Za-z0-9]+)", "storage_query_target_health": r"[-]+\s+([a-z0-9-]+)\s+[-]+\s+|Devices\s+|UUID:([a-z0-9-]+)\s+" r"Targets:\[([0-9 ]+)\]\s+Rank:(\d+)\s+State:(\w+)|" r"(?:Read\s+Errors|Write\s+Errors|Unmap\s+Errors|Checksum\s+Errors|" r"Error\s+Log\s+Entries|Media\s+Errors|Temperature|" r"Available\s+Spare|Device\s+Reliability|Read\s+Only|" r"Volatile\s+Memory\s+Backup):\s?([A-Za-z0-9- ]+)", "storage_set_faulty": r"[-]+\s+([a-z0-9-]+)\s+[-]+\s+|Devices\s+|(?:UUID:[a-z0-9-]+\s+" r"Targets:\[[0-9 ]+\]\s+Rank:\d+\s+State:(\w+))", "system_query": r"(\d+|\[[0-9-,]+\])\s+([A-Za-z]+)", "system_start": r"(\d+|\[[0-9-,]+\])\s+([A-Za-z]+)\s+([A-Za-z]+)", "system_stop": r"(\d+|\[[0-9-,]+\])\s+([A-Za-z]+)\s+([A-Za-z]+)", } def __init__(self, path, yaml_cfg=None): """Create a dmg Command object. Args: path (str): path to the dmg command yaml_cfg (DmgYamlParameters, optional): dmg config file settings. Defaults to None, in which case settings must be supplied as command-line parameters. """ super(DmgCommandBase, self).__init__("/run/dmg/*", "dmg", path, yaml_cfg) # If specified use the configuration file from the YamlParameters object default_yaml_file = None if isinstance(self.yaml, YamlParameters): default_yaml_file = self.yaml.filename self._hostlist = FormattedParameter("-l {}") self.hostfile = FormattedParameter("-f {}") self.configpath = FormattedParameter("-o {}", default_yaml_file) self.insecure = FormattedParameter("-i", False) self.debug = FormattedParameter("-d", False) self.json = FormattedParameter("-j", False) @property def hostlist(self): """Get the hostlist that was set. Returns a string list. """ if self.yaml: hosts = self.yaml.hostlist.value else: hosts = self._hostlist.value.split(",") return hosts @hostlist.setter def hostlist(self, hostlist): """Set the hostlist to be used for dmg invocation. Args: hostlist (string list): list of host addresses """ if self.yaml: if not isinstance(hostlist, list): hostlist = hostlist.split(",") self.yaml.hostlist.update(hostlist, "dmg.yaml.hostlist") else: if isinstance(hostlist, list): hostlist = ",".join(hostlist) self._hostlist.update(hostlist, "dmg._hostlist") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg sub command object based upon the sub-command.""" if self.sub_command.value == "network": self.sub_command_class = self.NetworkSubCommand() elif self.sub_command.value == "pool": self.sub_command_class = self.PoolSubCommand() elif self.sub_command.value == "storage": self.sub_command_class = self.StorageSubCommand() elif self.sub_command.value == "system": self.sub_command_class = self.SystemSubCommand() else: self.sub_command_class = None class NetworkSubCommand(CommandWithSubCommand): """Defines an object for the dmg network sub command.""" def __init__(self): """Create a dmg network subcommand object.""" super(DmgCommandBase.NetworkSubCommand, self).__init__("/run/dmg/network/*", "network") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg network sub command object.""" if self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() else: self.sub_command_class = None class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg network scan command.""" def __init__(self): """Create a dmg network scan command object.""" super(DmgCommandBase.NetworkSubCommand.ScanSubCommand, self).__init__("/run/dmg/network/scan/*", "scan") self.provider = FormattedParameter("-p {}", None) self.all = FormattedParameter("-a", False) class PoolSubCommand(CommandWithSubCommand): """Defines an object for the dmg pool sub command.""" def __init__(self): """Create a dmg pool subcommand object.""" super(DmgCommandBase.PoolSubCommand, self).__init__("/run/dmg/pool/*", "pool") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg pool sub command object.""" if self.sub_command.value == "create": self.sub_command_class = self.CreateSubCommand() elif self.sub_command.value == "delete-acl": self.sub_command_class = self.DeleteAclSubCommand() elif self.sub_command.value == "destroy": self.sub_command_class = self.DestroySubCommand() elif self.sub_command.value == "get-acl": self.sub_command_class = self.GetAclSubCommand() elif self.sub_command.value == "list": self.sub_command_class = self.ListSubCommand() elif self.sub_command.value == "overwrite-acl": self.sub_command_class = self.OverwriteAclSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "set-prop": self.sub_command_class = self.SetPropSubCommand() elif self.sub_command.value == "update-acl": self.sub_command_class = self.UpdateAclSubCommand() elif self.sub_command.value == "exclude": self.sub_command_class = self.ExcludeSubCommand() elif self.sub_command.value == "reintegrate": self.sub_command_class = self.ReintegrateSubCommand() else: self.sub_command_class = None class CreateSubCommand(CommandWithParameters): """Defines an object for the dmg pool create command.""" def __init__(self): """Create a dmg pool create command object.""" super(DmgCommandBase.PoolSubCommand.CreateSubCommand, self).__init__("/run/dmg/pool/create/*", "create") self.group = FormattedParameter("--group={}", None) self.user = FormattedParameter("--user={}", None) self.acl_file = FormattedParameter("--acl-file={}", None) self.scm_size = FormattedParameter("--scm-size={}", None) self.nvme_size = FormattedParameter("--nvme-size={}", None) self.ranks = FormattedParameter("--ranks={}", None) self.nsvc = FormattedParameter("--nsvc={}", None) self.sys = FormattedParameter("--sys={}", None) class ExcludeSubCommand(CommandWithParameters): """Defines an object for the dmg pool exclude command.""" def __init__(self): """Create a dmg pool exclude command object.""" super(DmgCommandBase.PoolSubCommand.ExcludeSubCommand, self).__init__("/run/dmg/pool/exclude/*", "exclude") self.pool = FormattedParameter("--pool={}", None) self.rank = FormattedParameter("--rank={}", None) self.tgt_idx = FormattedParameter("--target-idx={}", None) class ReintegrateSubCommand(CommandWithParameters): """Defines an object for dmg pool reintegrate command.""" def __init__(self): """Create a dmg pool reintegrate command object.""" super(DmgCommandBase.PoolSubCommand.ReintegrateSubCommand, self).__init__("/run/dmg/pool/reintegrate/*", "reintegrate") self.pool = FormattedParameter("--pool={}", None) self.rank = FormattedParameter("--rank={}", None) self.tgt_idx = FormattedParameter("--target-idx={}", None) class DeleteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool delete-acl command.""" def __init__(self): """Create a dmg pool delete-acl command object.""" super(DmgCommandBase.PoolSubCommand.DeleteAclSubCommand, self).__init__("/run/dmg/pool/delete-acl/*", "delete-acl") self.pool = FormattedParameter("--pool={}", None) self.principal = FormattedParameter("-p {}", None) class DestroySubCommand(CommandWithParameters): """Defines an object for the dmg pool destroy command.""" def __init__(self): """Create a dmg pool destroy command object.""" super(DmgCommandBase.PoolSubCommand.DestroySubCommand, self).__init__("/run/dmg/pool/destroy/*", "destroy") self.pool = FormattedParameter("--pool={}", None) self.sys_name = FormattedParameter("--sys-name={}", None) self.force = FormattedParameter("--force", False) class GetAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool get-acl command.""" def __init__(self): """Create a dmg pool get-acl command object.""" super(DmgCommandBase.PoolSubCommand.GetAclSubCommand, self).__init__("/run/dmg/pool/get-acl/*", "get-acl") self.pool = FormattedParameter("--pool={}", None) class ListSubCommand(CommandWithParameters): """Defines an object for the dmg pool list command.""" def __init__(self): """Create a dmg pool list command object.""" super(DmgCommandBase.PoolSubCommand.ListSubCommand, self).__init__("/run/dmg/pool/list/*", "list") class OverwriteAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool overwrite-acl command.""" def __init__(self): """Create a dmg pool overwrite-acl command object.""" super(DmgCommandBase.PoolSubCommand.OverwriteAclSubCommand, self).__init__("/run/dmg/pool/overwrite-acl/*", "overwrite-acl") self.pool = FormattedParameter("--pool={}", None) self.acl_file = FormattedParameter("-a {}", None) class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg pool query command.""" def __init__(self): """Create a dmg pool query command object.""" super(DmgCommandBase.PoolSubCommand.QuerySubCommand, self).__init__("/run/dmg/pool/query/*", "query") self.pool = FormattedParameter("--pool={}", None) class SetPropSubCommand(CommandWithParameters): """Defines an object for the dmg pool set-prop command.""" def __init__(self): """Create a dmg pool set-prop command object.""" super(DmgCommandBase.PoolSubCommand.SetPropSubCommand, self).__init__("/run/dmg/pool/set-prop/*", "set-prop") self.pool = FormattedParameter("--pool={}", None) self.name = FormattedParameter("--name={}", None) self.value = FormattedParameter("--value={}", None) class UpdateAclSubCommand(CommandWithParameters): """Defines an object for the dmg pool update-acl command.""" def __init__(self): """Create a dmg pool update-acl command object.""" super(DmgCommandBase.PoolSubCommand.UpdateAclSubCommand, self).__init__("/run/dmg/pool/update-acl/*", "update-acl") self.pool = FormattedParameter("--pool={}", None) self.acl_file = FormattedParameter("-a {}", None) self.entry = FormattedParameter("-e {}", None) class StorageSubCommand(CommandWithSubCommand): """Defines an object for the dmg storage sub command.""" def __init__(self): """Create a dmg storage subcommand object.""" super(DmgCommandBase.StorageSubCommand, self).__init__("/run/dmg/storage/*", "storage") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg storage sub command object.""" if self.sub_command.value == "format": self.sub_command_class = self.FormatSubCommand() elif self.sub_command.value == "prepare": self.sub_command_class = self.PrepareSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "scan": self.sub_command_class = self.ScanSubCommand() elif self.sub_command.value == "set": self.sub_command_class = self.SetSubCommand() else: self.sub_command_class = None class FormatSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage format command object.""" super(DmgCommandBase.StorageSubCommand.FormatSubCommand, self).__init__("/run/dmg/storage/format/*", "format") self.reformat = FormattedParameter("--reformat", False) class PrepareSubCommand(CommandWithParameters): """Defines an object for the dmg storage format command.""" def __init__(self): """Create a dmg storage prepare command object.""" super(DmgCommandBase.StorageSubCommand.PrepareSubCommand, self).__init__("/run/dmg/storage/prepare/*", "prepare") self.pci_whitelist = FormattedParameter("-w {}", None) self.hugepages = FormattedParameter("-p {}", None) self.target_user = FormattedParameter("-u {}", None) self.nvme_only = FormattedParameter("-n", False) self.scm_only = FormattedParameter("-s", False) self.reset = FormattedParameter("--reset", False) self.force = FormattedParameter("-f", False) class QuerySubCommand(CommandWithSubCommand): """Defines an object for the dmg query format command.""" def __init__(self): """Create a dmg storage query command object.""" super(DmgCommandBase.StorageSubCommand.QuerySubCommand, self).__init__("/run/dmg/storage/query/*", "query") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg pool sub command object.""" if self.sub_command.value == "nvme-health": self.sub_command_class = self.NvmeHealthSubCommand() elif self.sub_command.value == "target-health": self.sub_command_class = self.TargetHealthSubCommand() elif self.sub_command.value == "device-health": self.sub_command_class = self.DeviceHealthSubCommand() elif self.sub_command.value == "list-devices": self.sub_command_class = self.ListDevicesSubCommand() elif self.sub_command.value == "list-pools": self.sub_command_class = self.ListPoolsSubCommand() else: self.sub_command_class = None class NvmeHealthSubCommand(CommandWithParameters): """Defines a dmg storage query nvme-health object.""" def __init__(self): """Create a dmg storage query nvme-health object.""" super( DmgCommandBase.StorageSubCommand.QuerySubCommand. NvmeHealthSubCommand, self).__init__("/run/dmg/storage/query/nvme-health/*", "nvme-health") class TargetHealthSubCommand(CommandWithParameters): """Defines a dmg storage query target-health object.""" def __init__(self): """Create a dmg storage query target-health object.""" super( DmgCommandBase.StorageSubCommand.QuerySubCommand. TargetHealthSubCommand, self).__init__( "/run/dmg/storage/query/target-health/*", "target-health") self.rank = FormattedParameter("-r {}", None) self.tgtid = FormattedParameter("-t {}", None) class DeviceHealthSubCommand(CommandWithParameters): """Defines a dmg storage query device-health object.""" def __init__(self): """Create a dmg storage query device-health object.""" super( DmgCommandBase.StorageSubCommand.QuerySubCommand. DeviceHealthSubCommand, self).__init__( "/run/dmg/storage/query/device-health/*", "device-health") self.uuid = FormattedParameter("-u {}", None) class ListDevicesSubCommand(CommandWithParameters): """Defines a dmg storage query list-devices object.""" def __init__(self): """Create a dmg storage query list-devices object.""" super( DmgCommandBase.StorageSubCommand.QuerySubCommand. ListDevicesSubCommand, self).__init__("/run/dmg/storage/query/list-devices/*", "list-devices") self.rank = FormattedParameter("-r {}", None) self.uuid = FormattedParameter("-u {}", None) self.health = FormattedParameter("-b", False) class ListPoolsSubCommand(CommandWithParameters): """Defines a dmg storage query list-pools object.""" def __init__(self): """Create a dmg storage query list-pools object.""" super( DmgCommandBase.StorageSubCommand.QuerySubCommand. ListPoolsSubCommand, self).__init__("/run/dmg/storage/query/list-pools/*", "list-pools") self.rank = FormattedParameter("-r {}", None) self.uuid = FormattedParameter("-u {}", None) self.verbose = FormattedParameter("--verbose", False) class ScanSubCommand(CommandWithParameters): """Defines an object for the dmg storage scan command.""" def __init__(self): """Create a dmg storage scan command object.""" super(DmgCommandBase.StorageSubCommand.ScanSubCommand, self).__init__("/run/dmg/storage/scan/*", "scan") self.summary = FormattedParameter("-m", False) self.verbose = FormattedParameter("--verbose", False) class SetSubCommand(CommandWithSubCommand): """Defines an object for the dmg storage set command.""" def __init__(self): """Create a dmg storage set command object.""" super(DmgCommandBase.StorageSubCommand.SetSubCommand, self).__init__("/run/dmg/storage/set/*", "set") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg set sub command object.""" if self.sub_command.value == "nvme-faulty": self.sub_command_class = self.NvmeFaultySubCommand() else: self.sub_command_class = None class NvmeFaultySubCommand(CommandWithParameters): """Defines a dmg storage set nvme-faulty object.""" def __init__(self): """Create a dmg storage set nvme-faulty object.""" super( DmgCommandBase.StorageSubCommand.SetSubCommand. NvmeFaultySubCommand, self).__init__("/run/dmg/storage/query/device-state/*", "nvme-faulty") self.uuid = FormattedParameter("-u {}", None) self.force = FormattedParameter("--force", False) class SystemSubCommand(CommandWithSubCommand): """Defines an object for the dmg system sub command.""" def __init__(self): """Create a dmg system subcommand object.""" super(DmgCommandBase.SystemSubCommand, self).__init__("/run/dmg/system/*", "system") def get_sub_command_class(self): # pylint: disable=redefined-variable-type """Get the dmg system sub command object.""" if self.sub_command.value == "leader-query": self.sub_command_class = self.LeaderQuerySubCommand() elif self.sub_command.value == "list-pools": self.sub_command_class = self.ListPoolsSubCommand() elif self.sub_command.value == "query": self.sub_command_class = self.QuerySubCommand() elif self.sub_command.value == "start": self.sub_command_class = self.StartSubCommand() elif self.sub_command.value == "stop": self.sub_command_class = self.StopSubCommand() else: self.sub_command_class = None class LeaderQuerySubCommand(CommandWithParameters): """Defines an object for the dmg system leader-query command.""" def __init__(self): """Create a dmg system leader-query command object.""" super(DmgCommandBase.SystemSubCommand.LeaderQuerySubCommand, self).__init__("/run/dmg/system/leader-query/*", "leader-query") class ListPoolsSubCommand(CommandWithParameters): """Defines an object for the dmg system list-pools command.""" def __init__(self): """Create a dmg system list-pools command object.""" super(DmgCommandBase.SystemSubCommand.ListPoolsSubCommand, self).__init__("/run/dmg/system/list-pools/*", "list-pools") class QuerySubCommand(CommandWithParameters): """Defines an object for the dmg system query command.""" def __init__(self): """Create a dmg system query command object.""" super(DmgCommandBase.SystemSubCommand.QuerySubCommand, self).__init__("/run/dmg/system/query/*", "query") self.rank = FormattedParameter("--rank={}") self.verbose = FormattedParameter("--verbose", False) class StartSubCommand(CommandWithParameters): """Defines an object for the dmg system start command.""" def __init__(self): """Create a dmg system start command object.""" super(DmgCommandBase.SystemSubCommand.StartSubCommand, self).__init__("/run/dmg/system/start/*", "start") class StopSubCommand(CommandWithParameters): """Defines an object for the dmg system stop command.""" def __init__(self): """Create a dmg system stop command object.""" super(DmgCommandBase.SystemSubCommand.StopSubCommand, self).__init__("/run/dmg/system/stop/*", "stop") self.force = FormattedParameter("--force", False)