def __init__(self, hosts, tmp):
"""Create a dcp Command object."""
super().__init__("/run/dcp/*", "dcp", hosts, tmp)
# dcp options
# IO buffer size in bytes (default 64MB)
self.blocksize = FormattedParameter("--blocksize {}")
# New versions use bufsize instead of blocksize
self.bufsize = FormattedParameter("--bufsize {}")
# work size per task in bytes (default 64MB)
self.chunksize = FormattedParameter("--chunksize {}")
# DAOS prefix for unified namespace path
self.daos_prefix = FormattedParameter("--daos-prefix {}")
# DAOS API in {DFS, DAOS} (default uses DFS for POSIX containers)
self.daos_api = FormattedParameter("--daos-api {}")
# read source list from file
self.input_file = FormattedParameter("--input {}")
# copy original files instead of links
self.dereference = FormattedParameter("--dereference", False)
# don't follow links in source
self.no_dereference = FormattedParameter("--no-dereference", False)
# preserve permissions, ownership, timestamps, extended attributes
self.preserve = FormattedParameter("--preserve", False)
# open files with O_DIRECT
self.direct = FormattedParameter("--direct", False)
# create sparse files when possible
self.sparse = FormattedParameter("--sparse", False)
# print progress every N seconds
self.progress = FormattedParameter("--progress {}")
# verbose output
self.verbose = FormattedParameter("--verbose", False)
# quiet output
self.quiet = FormattedParameter("--quiet", False)
# print help/usage
self.print_usage = FormattedParameter("--help", False)
# source path
self.src_path = BasicParameter(None)
# destination path
self.dst_path = BasicParameter(None)
def __init__(self, log_dir="/tmp"):
"""Initialize a TransportConfig object."""
super(DaosAgentTransportCredentials,
self).__init__("/run/agent_config/transport_config/*",
"transport_config", log_dir)
# Additional daos_agent transport credential parameters:
# - server_name: <str>, e.g. "daos_server"
# Name of server accodring to its certificate [daos_agent only]
#
self.server_name = BasicParameter(None, None)
self.cert = LogParameter(log_dir, None, "agent.crt")
self.key = LogParameter(log_dir, None, "agent.key")
def __init__(self, filename, common_yaml):
"""Initialize an DaosAgentYamlParameters object.
Args:
filename (str): yaml configuration file name
common_yaml (YamlParameters): [description]
"""
super().__init__("/run/agent_config/*", filename, None, common_yaml)
# All log files should be placed in the same directory on each host to
# enable easy log file archiving by launch.py
log_dir = os.environ.get("DAOS_TEST_LOG_DIR", "/tmp")
# daos_agent parameters:
# - runtime_dir: <str>, e.g. /var/run/daos_agent
# Use the given directory for creating unix domain sockets
# - log_file: <str>, e.g. /tmp/daos_agent.log
# Full path and name of the DAOS agent logfile.
# - control_log_mask: <str>, one of: error, info, debug
# Specifies the log level for agent logs.
self.runtime_dir = BasicParameter(None, "/var/run/daos_agent")
self.log_file = LogParameter(log_dir, None, "daos_agent.log")
self.control_log_mask = BasicParameter(None, "debug")
def __init__(self, namespace, command, path="", subprocess=False):
"""Create a CommandWithSubCommand object.
Args:
namespace (str): yaml namespace (path to parameters)
command (str): string of the command to be executed.
path (str, optional): path to location of command binary file.
Defaults to "".
subprocess (bool, optional): whether the command is run as a
subprocess. Defaults to False.
"""
super(CommandWithSubCommand, self).__init__(namespace, command, path)
# Define the sub-command parameter whose value is used to assign the
# sub-command's CommandWithParameters-based class. Use the command to
# create uniquely named yaml parameter names.
#
# This parameter can be specified in the test yaml like so:
# <command>:
# <command>_sub_command: <sub_command>
# <sub_command>:
# <sub_command>_sub_command: <sub_command_sub_command>
#
self.sub_command = BasicParameter(None,
yaml_key="{}_sub_command".format(
self._command))
# Define the class to represent the active sub-command and it's specific
# parameters. Multiple sub-commands may be available, but only one can
# be active at a given time.
#
# The self.get_sub_command_class() method is called after obtaining the
# main command's parameter values, in self.get_params(), to assign the
# sub-command's class. This is typically a class based upon the
# CommandWithParameters class, but can be any object with a __str__()
# method (including a simple str object).
#
self.sub_command_class = None
# Define an attribute to store the CmdResult from the last run() call.
# A CmdResult object has the following properties:
# command - command string
# exit_status - exit_status of the command
# stdout - the stdout
# stderr - the stderr
# duration - command execution time
# interrupted - whether the command completed within timeout
# pid - command's pid
self.result = None
def __init__(self, namespace, command):
"""Create a daos-deserialize Command object."""
super().__init__(namespace, command)
# daos-deserialize options
# pool uuid for containers
self.pool = FormattedParameter("--pool {}")
# verbose output
self.verbose = FormattedParameter("--verbose", False)
# quiet output
self.quiet = FormattedParameter("--quiet", False)
# print help/usage
self.print_usage = FormattedParameter("--help", False)
# source path
self.src_path = BasicParameter(None)
def __init__(self, namespace, command):
"""Create a daos-serialize Command object."""
super().__init__(namespace, command)
# daos-serialize options
# path to output serialized hdf5 files
self.output_path = FormattedParameter("--output-path {}")
# verbose output
self.verbose = FormattedParameter("--verbose", False)
# quiet output
self.quiet = FormattedParameter("--quiet", False)
# print help/usage
self.print_usage = FormattedParameter("--help", False)
# source path
self.src_path = BasicParameter(None)
def __init__(self, namespace, command, path="", timeout=10):
"""Create a SubProcessCommand object.
Args:
namespace (str): yaml namespace (path to parameters)
command (str): string of the command to be executed.
path (str, optional): path to location of command binary file.
Defaults to "".
timeout (int, optional): number of seconds to wait for patterns to
appear in the subprocess output. Defaults to 10 seconds.
"""
super(SubProcessCommand, self).__init__(namespace, command, path, True)
# Attributes used to determine command success when run as a subprocess
# See self.check_subprocess_status() for details.
self.pattern = None
self.pattern_count = 1
self.pattern_timeout = BasicParameter(timeout, timeout)
def __init__(self, path="", filename="testfile", env=None):
"""Create a ExecutableCommand object.
Uses Avocado's utils.process module to run a command str provided.
Args:
command (str): string of the command to be executed.
path (str, optional): path to location of command binary file. Defaults to ""
"""
super().__init__("/run/gen_io_conf/*", "daos_gen_io_conf", path)
self.verbose = True
self.env = env
self.ranks = FormattedParameter("-g {}")
self.targets = FormattedParameter("-t {}")
self.obj_num = FormattedParameter("-o {}")
self.akeys = FormattedParameter("-a {}")
self.dkeys = FormattedParameter("-d {}")
self.record_size = FormattedParameter("-s {}")
self.obj_class = FormattedParameter("-O {}")
self.filename = BasicParameter(None, filename)
def __init__(self, namespace, command):
"""Create a dbench Command object."""
super().__init__(namespace, command)
# dbench options
self.timelimit = FormattedParameter("--timelimit {}")
self.loadfile = FormattedParameter("--loadfile {}")
self.directory = FormattedParameter("--directory {}")
self.tcp_options = FormattedParameter("--tcp-options {}")
self.target_rate = FormattedParameter("--target-rate {}")
self.sync = FormattedParameter("--sync", False)
self.fsync = FormattedParameter("--fsync", False)
self.xattr = FormattedParameter("--xattr", False)
self.no_resolve = FormattedParameter("--no-resolve", False)
self.clients_per_process = FormattedParameter(
"--clients-per-process {}")
self.one_byte_write_fix = FormattedParameter("--one-byte-write-fix",
False)
self.stat_check = FormattedParameter("--stat-check", False)
self.fake_io = FormattedParameter("--fake-io", False)
self.skip_cleanup = FormattedParameter("--skip-cleanup", False)
self.per_client_results = FormattedParameter("--per-client-results",
False)
self.num_of_procs = BasicParameter(None)
def __init__(self, filename, common_yaml):
"""Initialize an DaosServerYamlParameters object.
Args:
filename (str): yaml configuration file name
common_yaml (YamlParameters): [description]
"""
super().__init__("/run/server_config/*", filename, None, common_yaml)
# daos_server configuration file parameters
#
# - provider: <str>, e.g. ofi+verbs;ofi_rxm
# Force a specific provider to be used by all the servers.
#
# - hyperthreads: <bool>, e.g. True
# When Hyperthreading is enabled and supported on the system, this
# parameter defines whether the DAOS service thread should only be
# bound to different physical cores (False) or hyperthreads (True)
#
# - socket_dir: <str>, e.g. /var/run/daos_server
# DAOS Agent and DAOS Server both use unix domain sockets for
# communication with other system components. This setting is the
# base location to place the sockets in.
#
# - nr_hugepages: <int>, e.g. 4096
# Number of hugepages to allocate for use by NVMe SSDs
#
# - control_log_mask: <str>, e.g. DEBUG
# Force specific debug mask for daos_server (control plane).
#
# - control_log_file: <str>, e.g. /tmp/daos_control.log
# Force specific path for daos_server (control plane) logs.
#
# - user_name: <str>, e.g. daosuser
# Username used to lookup user uid/gid to drop privileges to if
# started as root. After control plane start-up and configuration,
# before starting data plane, process ownership will be dropped to
# those of supplied user.
#
# - group_name: <str>, e.g. daosgroup
# Group name used to lookup group gid to drop privileges to when
# user_name is root. If user is a member of group, this group gid
# is set for the running process. If group look up fails or user
# is not member, use uid return from user lookup.
#
default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+sockets")
# All log files should be placed in the same directory on each host to
# enable easy log file archiving by launch.py
log_dir = os.environ.get("DAOS_TEST_LOG_DIR", "/tmp")
self.provider = BasicParameter(None, default_provider)
self.hyperthreads = BasicParameter(None, False)
self.socket_dir = BasicParameter(None, "/var/run/daos_server")
self.nr_hugepages = BasicParameter(None, 4096)
self.control_log_mask = BasicParameter(None, "DEBUG")
self.control_log_file = LogParameter(log_dir, None, "daos_control.log")
self.helper_log_file = LogParameter(log_dir, None, "daos_admin.log")
self.telemetry_port = BasicParameter(None, 9191)
default_enable_vmd_val = os.environ.get("DAOS_ENABLE_VMD", "False")
default_enable_vmd = ast.literal_eval(default_enable_vmd_val)
self.enable_vmd = BasicParameter(None, default_enable_vmd)
# Used to drop privileges before starting data plane
# (if started as root to perform hardware provisioning)
self.user_name = BasicParameter(None)
self.group_name = BasicParameter(None)
# Defines the number of single engine config parameters to define in
# the yaml file
self.engines_per_host = BasicParameter(None)
# Single engine config parameters. Default to one set of I/O Engine
# parameters - for the config_file_gen.py tool. Calling get_params()
# will update the list to match the number of I/O Engines requested by
# the self.engines_per_host.value.
self.engine_params = [self.PerEngineYamlParameters()]
self.fault_path = BasicParameter(None)
def __init__(self, index=None, provider=None):
"""Create a SingleServerConfig object.
Args:
index (int, optional): index number for the namespace path used
when specifying multiple servers per host. Defaults to None.
"""
namespace = "/run/server_config/servers/*"
if isinstance(index, int):
namespace = "/run/server_config/servers/{}/*".format(index)
super().__init__(namespace)
if provider is not None:
self._provider = provider
else:
self._provider = os.environ.get("CRT_PHY_ADDR_STR",
"ofi+sockets")
# Use environment variables to get default parameters
default_interface = os.environ.get("DAOS_TEST_FABRIC_IFACE",
"eth0")
default_port = int(os.environ.get("OFI_PORT", 31416))
default_share_addr = int(os.environ.get("CRT_CTX_SHARE_ADDR", 0))
# All log files should be placed in the same directory on each host
# to enable easy log file archiving by launch.py
log_dir = os.environ.get("DAOS_TEST_LOG_DIR", "/tmp")
# Parameters
# targets: I/O service threads per engine
# first_core: starting index for targets
# nr_xs_helpers: I/O offload threads per engine
# fabric_iface: map to OFI_INTERFACE=eth0
# fabric_iface_port: map to OFI_PORT=31416
# log_mask: map to D_LOG_MASK env
# log_file: map to D_LOG_FILE env
# env_vars: influences DAOS I/O Engine behavior
# Add to enable scalable endpoint:
# - CRT_CTX_SHARE_ADDR=1
# - CRT_CTX_NUM=8
self.targets = BasicParameter(None, 8)
self.first_core = BasicParameter(None, 0)
self.nr_xs_helpers = BasicParameter(None, 4)
self.fabric_iface = BasicParameter(None, default_interface)
self.fabric_iface_port = BasicParameter(None, default_port)
self.pinned_numa_node = BasicParameter(None)
self.log_mask = BasicParameter(None, "INFO")
self.log_file = LogParameter(log_dir, None, "daos_server.log")
# Set default environment variables
default_env_vars = [
"ABT_ENV_MAX_NUM_XSTREAMS=100",
"ABT_MAX_NUM_XSTREAMS=100",
"DAOS_MD_CAP=1024",
"DD_MASK=mgmt,io,md,epc,rebuild",
]
default_env_vars.extend(self.REQUIRED_ENV_VARS["common"])
for name in self._provider.split(";"):
if name in self.REQUIRED_ENV_VARS:
default_env_vars.extend(self.REQUIRED_ENV_VARS[name])
self.env_vars = BasicParameter(None, default_env_vars)
# global CRT_CTX_SHARE_ADDR shared with client
self.crt_ctx_share_addr = BasicParameter(None, default_share_addr)
# global CRT_TIMEOUT shared with client
self.crt_timeout = BasicParameter(None, 30)
# Storage definition parameters:
#
# When scm_class is set to ram, tmpfs will be used to emulate SCM.
# scm_mount: /mnt/daos - map to -s /mnt/daos
# scm_class: ram
# scm_size: 6 - size in GB units
#
# When scm_class is set to dcpm, scm_list is the list of device
# paths for AppDirect pmem namespaces (currently only one per
# server supported).
# scm_class: dcpm
# scm_list: [/dev/pmem0]
#
# If using NVMe SSD (will write /mnt/daos/daos_nvme.conf and start
# I/O service with -n <path>)
# bdev_class: nvme
# bdev_list: ["0000:81:00.0"] - generate regular nvme.conf
#
# If emulating NVMe SSD with malloc devices
# bdev_class: malloc - map to VOS_BDEV_CLASS=MALLOC
# bdev_size: 4 - malloc size of each device in GB.
# bdev_number: 1 - generate nvme.conf as follows:
# [Malloc]
# NumberOfLuns 1
# LunSizeInMB 4000
#
# If emulating NVMe SSD over kernel block device
# bdev_class: kdev - map to VOS_BDEV_CLASS=AIO
# bdev_list: [/dev/sdc] - generate nvme.conf as follows:
# [AIO]
# AIO /dev/sdc AIO2
#
# If emulating NVMe SSD with backend file
# bdev_class: file - map to VOS_BDEV_CLASS=AIO
# bdev_size: 16 - file size in GB. Create file if
# it does not exist.
# bdev_list: [/tmp/daos-bdev] - generate nvme.conf as follows:
# [AIO]
# AIO /tmp/aiofile AIO1 4096
self.scm_mount = BasicParameter(None, "/mnt/daos")
self.scm_class = BasicParameter(None, "ram")
self.scm_size = BasicParameter(None, 16)
self.scm_list = BasicParameter(None)
self.bdev_class = BasicParameter(None)
self.bdev_list = BasicParameter(None)
self.bdev_size = BasicParameter(None)
self.bdev_number = BasicParameter(None)
def __init__(self):
"""Initialize a RebuildTestParams object."""
super(RebuildTestParams, self).__init__("/run/rebuild/*")
self.object_class = BasicParameter(None)
self.rank = BasicParameter(None)
def __init__(self, pool, cb_handler=None, daos_command=None):
"""Create a TeestContainer object.
Args:
pool (TestPool): the test pool in which to create the container
cb_handler (CallbackHandler, optional): callback object to use with
the API methods. Defaults to None.
"""
super(TestContainer, self).__init__("/run/container/*", cb_handler)
self.pool = pool
self.object_qty = BasicParameter(None)
self.record_qty = BasicParameter(None)
self.akey_size = BasicParameter(None)
self.dkey_size = BasicParameter(None)
self.data_size = BasicParameter(None)
self.data_array_size = BasicParameter(0, 0)
# Provider access to get input params values
# for enabling different container properties
self.input_params = DaosInputParams()
# Optional daos command object to use with the USE_DAOS control method
self.daos = daos_command
# Optional daos command argument values to use with the USE_DAOS control
# method when creating/destroying containers
self.path = BasicParameter(None)
self.type = BasicParameter(None)
self.oclass = BasicParameter(None)
self.chunk_size = BasicParameter(None)
self.properties = BasicParameter(None)
self.daos_timeout = BasicParameter(None)
self.container = None
self.uuid = None
self.info = None
self.opened = False
self.written_data = []
self.epoch = None
def __init__(self, index=None):
"""Create a SingleServerConfig object.
Args:
index (int, optional): index number for the namespace path used
when specifying multiple servers per host. Defaults to None.
"""
namespace = "/run/server_config/servers/*"
if isinstance(index, int):
namespace = "/run/server_config/servers/{}/*".format(index)
super(
DaosServerYamlParameters.PerServerYamlParameters,
self).__init__(namespace)
# Use environment variables to get default parameters
default_interface = os.environ.get("OFI_INTERFACE", "eth0")
default_port = int(os.environ.get("OFI_PORT", 31416))
# All log files should be placed in the same directory on each host
# to enable easy log file archiving by launch.py
log_dir = os.environ.get("DAOS_TEST_LOG_DIR", "/tmp")
# Parameters
# targets: count of VOS targets
# first_core: starting index for targets
# nr_xs_helpers: offload helpers per server
# fabric_iface: map to OFI_INTERFACE=eth0
# fabric_iface_port: map to OFI_PORT=31416
# log_mask: map to D_LOG_MASK env
# log_file: map to D_LOG_FILE env
# env_vars: influences DAOS IO Server behaviour
# Add to enable scalable endpoint:
# - CRT_CREDIT_EP_CTX=0
# - CRT_CTX_SHARE_ADDR=1
# - CRT_CTX_NUM=8
# nvme options:
# - IO_STAT_PERIOD=10
self.targets = BasicParameter(None, 8)
self.first_core = BasicParameter(None, 0)
self.nr_xs_helpers = BasicParameter(None, 16)
self.fabric_iface = BasicParameter(None, default_interface)
self.fabric_iface_port = BasicParameter(None, default_port)
self.pinned_numa_node = BasicParameter(None)
self.log_mask = BasicParameter(None, "DEBUG,RPC=ERR")
self.log_file = LogParameter(log_dir, None, "daos_server.log")
self.env_vars = BasicParameter(
None,
["ABT_ENV_MAX_NUM_XSTREAMS=100",
"ABT_MAX_NUM_XSTREAMS=100",
"DAOS_MD_CAP=1024",
"CRT_CTX_SHARE_ADDR=0",
"CRT_TIMEOUT=30",
"FI_SOCKETS_MAX_CONN_RETRY=1",
"FI_SOCKETS_CONN_TIMEOUT=2000",
"DD_MASK=mgmt,io,md,epc,rebuild"]
)
# Storage definition parameters:
#
# When scm_class is set to ram, tmpfs will be used to emulate SCM.
# scm_mount: /mnt/daos - map to -s /mnt/daos
# scm_class: ram
# scm_size: 6 - size in GB units
#
# When scm_class is set to dcpm, scm_list is the list of device
# paths for AppDirect pmem namespaces (currently only one per
# server supported).
# scm_class: dcpm
# scm_list: [/dev/pmem0]
#
# If using NVMe SSD (will write /mnt/daos/daos_nvme.conf and start
# I/O service with -n <path>)
# bdev_class: nvme
# bdev_list: ["0000:81:00.0"] - generate regular nvme.conf
#
# If emulating NVMe SSD with malloc devices
# bdev_class: malloc - map to VOS_BDEV_CLASS=MALLOC
# bdev_size: 4 - malloc size of each device in GB.
# bdev_number: 1 - generate nvme.conf as follows:
# [Malloc]
# NumberOfLuns 1
# LunSizeInMB 4000
#
# If emulating NVMe SSD over kernel block device
# bdev_class: kdev - map to VOS_BDEV_CLASS=AIO
# bdev_list: [/dev/sdc] - generate nvme.conf as follows:
# [AIO]
# AIO /dev/sdc AIO2
#
# If emulating NVMe SSD with backend file
# bdev_class: file - map to VOS_BDEV_CLASS=AIO
# bdev_size: 16 - file size in GB. Create file if
# it does not exist.
# bdev_list: [/tmp/daos-bdev] - generate nvme.conf as follows:
# [AIO]
# AIO /tmp/aiofile AIO1 4096
self.scm_mount = BasicParameter(None, "/mnt/daos")
self.scm_class = BasicParameter(None, "ram")
self.scm_size = BasicParameter(None, 16)
self.scm_list = BasicParameter(None)
self.bdev_class = BasicParameter(None)
self.bdev_list = BasicParameter(None)
self.bdev_size = BasicParameter(None)
self.bdev_number = BasicParameter(None)