def verify_pool_readwrite(self, svc, uuid, action, expect='Pass'): """Verify client is able to perform read or write on a pool. Args: svc (int): pool svc number. uuid (str): pool uuid number. action (str): read or write on pool. expect (str): expecting behavior pass or deny with RC -1001. Return: bool: pass or fail. """ deny_access = '-1001' daos_cmd = DaosCommand(self.bin) daos_cmd.exit_status_exception = False if action.lower() == "write": result = daos_cmd.container_create(pool=uuid, svc=svc) elif action.lower() == "read": result = daos_cmd.pool_query(pool=uuid, svc=svc) else: self.fail("##In verify_pool_readwrite, invalid action: {}".format( action)) self.log.info( " In verify_pool_readwrite %s.\n =daos_cmd.run() result:\n%s", action, result) self.verify_daos_pool_cont_result(result, action, expect, deny_access)
def create_pool_and_container(self): """method to create pool and container""" scm_size = self.params.get("scm_size", "/run/server/*/", 138000000) num_of_pool = self.params.get("num_of_pool", "/run/server/*/", 3) container_per_pool = self.params.get("container_per_pool", "/run/server/*/", 2) for _ in range(num_of_pool): dmg = self.get_dmg_command() result = dmg.pool_create(scm_size) uuid = result['uuid'] daos_cmd = DaosCommand(self.bin) for _ in range(container_per_pool): result = daos_cmd.container_create(pool=uuid) self.log.info("container create status: %s", result)
class MpiioTests(TestWithServers): """ Runs ROMIO, LLNL, MPI4PY and HDF5 test suites. :avocado: recursive """ def __init__(self, *args, **kwargs): """Initialize a TestWithServers object.""" super(MpiioTests, self).__init__(*args, **kwargs) self.hostfile_clients_slots = None self.mpio = None self.daos_cmd = None self.cont_uuid = None def setUp(self): super(MpiioTests, self).setUp() # initialize daos_cmd self.daos_cmd = DaosCommand(self.bin) # initialize a python pool object then create the underlying self.pool = TestPool(self.context, dmg_command=self.get_dmg_command()) self.pool.get_params(self) self.pool.create() def _create_cont(self): """Create a container. Args: daos_cmd (DaosCommand): daos command to issue the container create Returns: str: UUID of the created container """ cont_type = self.params.get("type", "/run/container/*") result = self.daos_cmd.container_create(pool=self.pool.uuid, svc=self.pool.svc_ranks, cont_type=cont_type) # Extract the container UUID from the daos container create output cont_uuid = re.findall("created\s+container\s+([0-9a-f-]+)", result.stdout) if not cont_uuid: self.fail("Error obtaining the container uuid from: {}".format( result.stdout)) self.cont_uuid = cont_uuid[0] def run_test(self, test_repo, test_name): """ Executable function to be used by test functions below test_repo --location of test repository test_name --name of the test to be run """ # initialize MpioUtils self.mpio = MpioUtils() if not self.mpio.mpich_installed(self.hostlist_clients): self.fail("Exiting Test: Mpich not installed") # initialize test specific variables client_processes = self.params.get("np", '/run/client_processes/') # create container self._create_cont() try: # running tests self.mpio.run_mpiio_tests(self.hostfile_clients, self.pool.uuid, self.pool.svc_ranks, test_repo, test_name, client_processes, self.cont_uuid) except MpioFailed as excep: self.fail("<{0} Test Failed> \n{1}".format(test_name, excep)) # Parsing output to look for failures # stderr directed to stdout stdout = os.path.join(self.logdir, "stdout") searchfile = open(stdout, "r") error_message = [ "non-zero exit code", "MPI_Abort", "MPI_ABORT", "ERROR" ] for line in searchfile: for error in error_message: if error in line: self.fail( "Test Failed with error_message: {}".format(error))
class FioBase(TestWithServers): """Base fio class. :avocado: recursive """ def __init__(self, *args, **kwargs): """Initialize a FioBase object.""" super(FioBase, self).__init__(*args, **kwargs) self.fio_cmd = None self.processes = None self.manager = None self.dfuse = None self.daos_cmd = None def setUp(self): """Set up each test case.""" # obtain separate logs self.update_log_file_names() # Start the servers and agents super(FioBase, self).setUp() # initialise daos_cmd self.daos_cmd = DaosCommand(self.bin) # Get the parameters for Fio self.fio_cmd = FioCommand() self.fio_cmd.get_params(self) self.processes = self.params.get("np", '/run/fio/client_processes/*') self.manager = self.params.get("manager", '/run/fio/*', "MPICH") def tearDown(self): """Tear down each test case.""" try: if self.dfuse: self.dfuse.stop() finally: # Stop the servers and agents super(FioBase, self).tearDown() def _create_pool(self): """Create a pool and execute Fio.""" # Get the pool params # pylint: disable=attribute-defined-outside-init self.pool = TestPool(self.context, dmg_command=self.get_dmg_command()) self.pool.get_params(self) # Create a pool self.pool.create() def _create_cont(self): """Create a container. Returns: str: UUID of the created container """ cont_type = self.params.get("type", "/run/container/*") result = self.daos_cmd.container_create(pool=self.pool.uuid, svc=self.pool.svc_ranks, cont_type=cont_type) # Extract the container UUID from the daos container create output cont_uuid = re.findall(r"created\s+container\s+([0-9a-f-]+)", result.stdout) if not cont_uuid: self.fail("Error obtaining the container uuid from: {}".format( result.stdout)) return cont_uuid[0] def _start_dfuse(self): """Create a DfuseCommand object to start dfuse.""" # Get Dfuse params self.dfuse = Dfuse(self.hostlist_clients, self.tmp) self.dfuse.get_params(self) # update dfuse params self.dfuse.set_dfuse_params(self.pool) self.dfuse.set_dfuse_cont_param(self._create_cont()) self.dfuse.set_dfuse_exports(self.server_managers[0], self.client_log) try: # start dfuse self.dfuse.run() except CommandFailure as error: self.log.error("Dfuse command %s failed on hosts %s", str(self.dfuse), str(NodeSet.fromlist(self.dfuse.hosts)), exc_info=error) self.fail("Unable to launch Dfuse.\n") def execute_fio(self): """Runner method for Fio.""" # Create a pool if one does not already exist if self.pool is None: self._create_pool() # start dfuse if api is POSIX if self.fio_cmd.api.value == "POSIX": # Connect to the pool, create container and then start dfuse # Uncomment below two lines once DAOS-3355 is resolved # self.pool.connect() # self.create_cont() self._start_dfuse() self.fio_cmd.update("global", "directory", self.dfuse.mount_dir.value, "fio --name=global --directory") # Run Fio self.fio_cmd.hosts = self.hostlist_clients self.fio_cmd.run() if self.dfuse: self.dfuse.stop() self.dfuse = None
class MdtestBase(TestWithServers): """Base mdtest class. :avocado: recursive """ def __init__(self, *args, **kwargs): """Initialize a MdtestBase object.""" super(MdtestBase, self).__init__(*args, **kwargs) self.mdtest_cmd = None self.processes = None self.hostfile_clients_slots = None self.dfuse = None self.daos_cmd = None def setUp(self): """Set up each test case.""" # obtain separate logs self.update_log_file_names() # Start the servers and agents super(MdtestBase, self).setUp() # initialize daos_cmd self.daos_cmd = DaosCommand(self.bin) # Get the parameters for Mdtest self.mdtest_cmd = MdtestCommand() self.mdtest_cmd.get_params(self) self.processes = self.params.get("np", '/run/mdtest/client_processes/*') self.manager = self.params.get("manager", '/run/mdtest/*', "MPICH") self.log.info('Clients %s', self.hostlist_clients) self.log.info('Servers %s', self.hostlist_servers) def tearDown(self): """Tear down each test case.""" try: if self.dfuse: self.dfuse.stop() finally: # Stop the servers and agents super(MdtestBase, self).tearDown() def create_pool(self): """Create a pool and execute Mdtest.""" # Get the pool params self.pool = TestPool(self.context, dmg_command=self.get_dmg_command()) self.pool.get_params(self) # Create a pool self.pool.create() def _create_cont(self): """Create a container. Returns: str: UUID of the created container """ cont_type = self.params.get("type", "/run/container/*") result = self.daos_cmd.container_create(pool=self.pool.uuid, svc=self.pool.svc_ranks, cont_type=cont_type) # Extract the container UUID from the daos container create output cont_uuid = re.findall(r"created\s+container\s+([0-9a-f-]+)", result.stdout) if not cont_uuid: self.fail("Error obtaining the container uuid from: {}".format( result.stdout)) return cont_uuid[0] def _start_dfuse(self): """Create a DfuseCommand object to start dfuse.""" # Get Dfuse params self.dfuse = Dfuse(self.hostlist_clients, self.tmp) self.dfuse.get_params(self) # update dfuse params self.dfuse.set_dfuse_params(self.pool) self.dfuse.set_dfuse_cont_param(self._create_cont()) self.dfuse.set_dfuse_exports(self.server_managers[0], self.client_log) try: # start dfuse self.dfuse.run() except CommandFailure as error: self.log.error("Dfuse command %s failed on hosts %s", str(self.dfuse), self.dfuse.hosts, exc_info=error) self.fail("Unable to launch Dfuse.\n") def execute_mdtest(self): """Runner method for Mdtest.""" # Create a pool if one does not already exist if self.pool is None: self.create_pool() # set Mdtest params self.mdtest_cmd.set_daos_params(self.server_group, self.pool) # start dfuse if api is POSIX if self.mdtest_cmd.api.value == "POSIX": # Connect to the pool, create container and then start dfuse # Uncomment below two lines once DAOS-3355 is resolved # self.pool.connect() # self.create_cont() self._start_dfuse() self.mdtest_cmd.test_dir.update(self.dfuse.mount_dir.value) # Run Mdtest self.run_mdtest(self.get_mdtest_job_manager_command(self.manager), self.processes) if self.dfuse: self.dfuse.stop() self.dfuse = None def get_mdtest_job_manager_command(self, manager): """Get the MPI job manager command for Mdtest. Returns: JobManager: the object for the mpi job manager command """ # Initialize MpioUtils if mdtest needs to be run using mpich if manager == "MPICH": mpio_util = MpioUtils() if mpio_util.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test: Mpich not installed") return Mpirun(self.mdtest_cmd, mpitype="mpich") return Orterun(self.mdtest_cmd) def run_mdtest(self, manager, processes): """Run the Mdtest command. Args: manager (str): mpi job manager command processes (int): number of host processes """ env = self.mdtest_cmd.get_default_env(str(manager), self.client_log) manager.assign_hosts(self.hostlist_clients, self.workdir, self.hostfile_clients_slots) manager.assign_processes(processes) manager.assign_environment(env) try: self.pool.display_pool_daos_space() manager.run() except CommandFailure as error: self.log.error("Mdtest Failed: %s", str(error)) self.fail("Test was expected to pass but it failed.\n") finally: self.pool.display_pool_daos_space()
class MpiioTests(TestWithServers): """Run ROMIO, LLNL, MPI4PY and HDF5 test suites. :avocado: recursive """ def __init__(self, *args, **kwargs): """Initialize a TestWithServers object.""" super().__init__(*args, **kwargs) self.hostfile_clients_slots = None self.mpio = None self.daos_cmd = None self.cont_uuid = None def setUp(self): super().setUp() # initialize daos_cmd self.daos_cmd = DaosCommand(self.bin) # initialize a python pool object then create the underlying self.pool = TestPool(self.context, self.get_dmg_command()) self.pool.get_params(self) self.pool.create() def _create_cont(self): """Create a container. Args: daos_cmd (DaosCommand): daos command to issue the container create Returns: str: UUID of the created container """ cont_type = self.params.get("type", "/run/container/*") result = self.daos_cmd.container_create(pool=self.pool.uuid, cont_type=cont_type) # Extract the container UUID from the daos container create output cont_uuid = re.findall(r"created\s+container\s+([0-9a-f-]+)", result.stdout_text) if not cont_uuid: self.fail("Error obtaining the container uuid from: {}".format( result.stdout_text)) self.cont_uuid = cont_uuid[0] def run_test(self, test_repo, test_name): """Execute function to be used by test functions below. test_repo --location of test repository test_name --name of the test to be run """ # Required to run daos command load_mpi("openmpi") # create container self._create_cont() # initialize MpioUtils self.mpio = MpioUtils() if not self.mpio.mpich_installed(self.hostlist_clients): self.fail("Exiting Test: Mpich not installed") # initialize test specific variables client_processes = self.params.get("np", '/run/client_processes/') try: # running tests result = self.mpio.run_mpiio_tests(self.hostfile_clients, self.pool.uuid, test_repo, test_name, client_processes, self.cont_uuid) except MpioFailed as excep: self.fail("<{0} Test Failed> \n{1}".format(test_name, excep)) # Check output for errors for output in (result.stdout_text, result.stderr_text): match = re.findall( r"(non-zero exit code|MPI_Abort|MPI_ABORT|ERROR)", output) if match: self.log.info( "The following error messages have been detected in the %s " "output:", test_name) for item in match: self.log.info(" %s", item) self.fail( "Error messages detected in {} output".format(test_name))