def get_ior_job_manager_command(self, custom_ior_cmd=None): """Get the MPI job manager command for IOR. Args: custom_ior_cmd (IorCommand): Custom IorCommand instance to create job_manager with. Returns: str: the path for the mpi job manager command """ # Initialize MpioUtils if IOR is running in MPIIO or DFS mode if self.ior_cmd.api.value in ["MPIIO", "POSIX", "DFS", "HDF5"]: mpio_util = MpioUtils() if mpio_util.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test: Mpich not installed") else: self.fail("Unsupported IOR API") if custom_ior_cmd: self.job_manager = Mpirun(custom_ior_cmd, self.subprocess, "mpich") else: self.job_manager = Mpirun(self.ior_cmd, self.subprocess, "mpich") return self.job_manager
def run_test(self, test_repo, test_name): """ Executable function to be used by test functions below test_repo --location of test repository test_name --name of the test to be run """ # initialize MpioUtils self.mpio = MpioUtils() if not self.mpio.mpich_installed(self.hostlist_clients): self.fail("Exiting Test: Mpich not installed") # initialise test specific variables client_processes = self.params.get("np", '/run/client_processes/') try: # running tests self.mpio.run_llnl_mpi4py_hdf5( self.hostfile_clients, self.pool.uuid, test_repo, test_name, client_processes) except MpioFailed as excep: self.fail("<{0} Test Failed> \n{1}".format(test_name, excep)) # Parsing output to look for failures # stderr directed to stdout stdout = os.path.join(self.logdir, "stdout") searchfile = open(stdout, "r") error_message = ["non-zero exit code", "MPI_Abort", "MPI_ABORT", "ERROR"] for line in searchfile: for error in error_message: if error in line: self.fail( "Test Failed with error_message: {}".format(error))
def ior_thread(self, pool, oclass, api, test, flags, results): """Start threads and wait until all threads are finished. Args: pool (object): pool handle oclass (str): IOR object class api (str): IOR api test (list): IOR test sequence flags (str): IOR flags results (queue): queue for returning thread results Returns: None """ processes = self.params.get("slots", "/run/ior/clientslots/*") container_info = {} mpio_util = MpioUtils() if mpio_util.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test : Mpich not installed on :" " {}".format(self.hostfile_clients[0])) self.pool = pool # Define the arguments for the ior_runner_thread method ior_cmd = IorCommand() ior_cmd.get_params(self) ior_cmd.set_daos_params(self.server_group, self.pool) ior_cmd.daos_oclass.update(oclass) ior_cmd.api.update(api) ior_cmd.transfer_size.update(test[2]) ior_cmd.block_size.update(test[3]) ior_cmd.flags.update(flags) container_info["{}{}{}" .format(oclass, api, test[2])] = str(uuid.uuid4()) # Define the job manager for the IOR command manager = Mpirun(ior_cmd, mpitype="mpich") manager.job.daos_cont.update(container_info ["{}{}{}".format(oclass, api, test[2])]) env = ior_cmd.get_default_env(str(manager)) manager.assign_hosts(self.hostlist_clients, self.workdir, None) manager.assign_processes(processes) manager.assign_environment(env, True) # run IOR Command try: manager.run() except CommandFailure as _error: results.put("FAIL")
def get_job_manager_command(self): """Get the MPI job manager command for IOR. Returns: str: the path for the mpi job manager command """ # Initialize MpioUtils if IOR is running in MPIIO mode if self.ior_cmd.api.value == "MPIIO": mpio_util = MpioUtils() if mpio_util.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test: Mpich not installed") return os.path.join(mpio_util.mpichinstall, "bin", "mpirun") return self.orterun
def run_test(self, test_repo, test_name): """Execute function to be used by test functions below. test_repo --absolute or relative (to self.mpichinstall) location of test repository test_name --name of the test to be run """ # Required to run daos command load_mpi("openmpi") # Create pool self.add_pool(connect=False) # create container self.add_container(self.pool) # initialize MpioUtils self.mpio = MpioUtils() if not self.mpio.mpich_installed(self.hostlist_clients): self.fail("Exiting Test: Mpich not installed") # fix up a relative test_repo specification if test_repo[0] != '/': test_repo = os.path.join(self.mpio.mpichinstall, test_repo) # initialize test specific variables client_processes = self.params.get("np", '/run/client_processes/') try: # running tests result = self.mpio.run_mpiio_tests(self.hostfile_clients, self.pool.uuid, test_repo, test_name, client_processes, self.container.uuid) except MpioFailed as excep: self.fail("<{0} Test Failed> \n{1}".format(test_name, excep)) # Check output for errors for output in (result.stdout_text, result.stderr_text): match = re.findall( r"(non-zero exit code|MPI_Abort|MPI_ABORT|ERROR)", output) if match: self.log.info( "The following error messages have been detected in the %s " "output:", test_name) for item in match: self.log.info(" %s", item) self.fail( "Error messages detected in {} output".format(test_name))
def get_mdtest_job_manager_command(self, manager): """Get the MPI job manager command for Mdtest. Returns: JobManager: the object for the mpi job manager command """ # Initialize MpioUtils if mdtest needs to be run using mpich if manager == "MPICH": mpio_util = MpioUtils() if mpio_util.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test: Mpich not installed") return Mpirun(self.mdtest_cmd, mpitype="mpich") return Orterun(self.mdtest_cmd)
def get_ior_job_manager_command(self): """Get the MPI job manager command for IOR. Returns: str: the path for the mpi job manager command """ # Initialize MpioUtils if IOR is running in MPIIO or DAOS mode if self.ior_cmd.api.value in ["MPIIO", "DAOS", "POSIX", "DFS"]: mpio_util = MpioUtils() if mpio_util.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test: Mpich not installed") else: self.fail("Unsupported IOR API") return Mpirun(self.ior_cmd, mpitype="mpich")
def ior_bg_thread(self, results): """Start IOR Background thread, This will write small data set and keep reading it in loop until it fails or main program exit. Args: results (queue): queue for returning thread results """ mpio_util = MpioUtils() if mpio_util.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test: Mpich not installed") # Define the IOR Command and use the parameter from yaml file. ior_bg_cmd = IorCommand() ior_bg_cmd.get_params(self) ior_bg_cmd.set_daos_params(self.server_group, self.pool) ior_bg_cmd.dfs_oclass.update(self.ior_cmd.dfs_oclass.value) ior_bg_cmd.api.update(self.ior_cmd.api.value) ior_bg_cmd.transfer_size.update(self.ior_scm_xfersize) ior_bg_cmd.block_size.update(self.ior_cmd.block_size.value) ior_bg_cmd.flags.update(self.ior_cmd.flags.value) ior_bg_cmd.test_file.update('/testfile_background') # Define the job manager for the IOR command self.job_manager = Mpirun(ior_bg_cmd, mpitype="mpich") self.create_cont() self.job_manager.job.dfs_cont.update(self.container.uuid) env = ior_bg_cmd.get_default_env(str(self.job_manager)) self.job_manager.assign_hosts(self.hostlist_clients, self.workdir, None) self.job_manager.assign_processes(1) self.job_manager.assign_environment(env, True) print('----Run IOR in Background-------') # run IOR Write Command try: self.job_manager.run() except (CommandFailure, TestFail) as _error: results.put("FAIL") return # run IOR Read Command in loop ior_bg_cmd.flags.update(self.ior_read_flags) while True: try: self.job_manager.run() except (CommandFailure, TestFail) as _error: results.put("FAIL") break
def executable(self, iorflags=None): """ Executable function to run ior for ssf and fpp """ # parameters used in pool create createmode = self.params.get("mode", '/run/pool/createmode/*/') createuid = os.geteuid() creategid = os.getegid() createsetid = self.params.get("setname", '/run/pool/createset/') createscm_size = self.params.get("scm_size", '/run/pool/createsize/') createnvme_size = self.params.get("nvme_size", '/run/pool/createsize/') createsvc = self.params.get("svcn", '/run/pool/createsvc/') iteration = self.params.get("iter", '/run/ior/iteration/') block_size = self.params.get("b", '/run/ior/transfersize_blocksize/*/') transfer_size = self.params.get("t", '/run/ior/transfersize_blocksize/*/') try: # initialize MpioUtils self.mpio = MpioUtils() if self.mpio.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test: Mpich not installed") #print self.mpio.mpichinstall # initialize a python pool object then create the underlying # daos storage self.pool = DaosPool(self.context) self.pool.create(createmode, createuid, creategid, createscm_size, createsetid, None, None, createsvc, createnvme_size) pool_uuid = self.pool.get_uuid_str() svc_list = "" for i in range(createsvc): svc_list += str(int(self.pool.svc.rl_ranks[i])) + ":" svc_list = svc_list[:-1] print("svc_list: {}".format(svc_list)) ior_utils.run_ior_mpiio(self.basepath, self.mpio.mpichinstall, pool_uuid, svc_list, self.num_procs, self.hostfile_clients, iorflags, iteration, transfer_size, block_size, True) except (DaosApiError, MpioFailed) as excep: print(excep)
def test_romio(self): """ Test ID: DAOS-1994 Run Romio test provided in mpich package Testing various I/O functions provided in romio test suite :avocado: tags=all,mpiio,pr,small,romio """ # setting romio parameters romio_test_repo = self.params.get("romio_repo", '/run/romio/') # initialize MpioUtils self.mpio = MpioUtils() if self.mpio.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test: Mpich not installed") try: # Romio do not need slots in hostfile with open(self.hostfile_clients) as client_file: new_text = client_file.read().replace('slots=1', '') with open(self.hostfile_clients, "w") as client_file: client_file.write(new_text) # running romio self.mpio.run_romio(self.basepath, self.hostlist_clients, romio_test_repo) # Parsing output to look for failures # stderr directed to stdout stdout = self.logdir + "/stdout" searchfile = open(stdout, "r") error_message = [ "non-zero exit code", "MPI_Abort", "errors", "failed to create pool", "failed to parse pool UUID", "failed to destroy pool" ] for line in searchfile: for i in xrange(len(error_message)): if error_message[i] in line: self.fail("Romio Test Failed with error_message: " "{}".format(error_message[i])) except (MpioFailed) as excep: self.fail("<Romio Test Failed> \n{}".format(excep))
def ior_thread(self, pool, oclass, api, test, flags, results): """This method calls job manager for IOR command invocation. Args: pool (object): pool handle oclass (str): IOR object class API (str): IOR API test (list): IOR test sequence flags (str): IOR flags results (queue): queue for returning thread results """ processes = self.params.get("slots", "/run/ior/clientslots/*") mpio_util = MpioUtils() if mpio_util.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test: Mpich not installed") self.pool = pool # Define the arguments for the ior_runner_thread method ior_cmd = IorCommand() ior_cmd.get_params(self) ior_cmd.set_daos_params(self.server_group, self.pool) ior_cmd.dfs_oclass.update(oclass) ior_cmd.api.update(api) ior_cmd.transfer_size.update(test[0]) ior_cmd.block_size.update(test[1]) ior_cmd.flags.update(flags) if "-w" in flags: self.container_info["{}{}{}" .format(oclass, api, test[0])] = str(uuid.uuid4()) # Define the job manager for the IOR command manager = Mpirun(ior_cmd, mpitype="mpich") key = "".join([oclass, api, str(test[0])]) manager.job.dfs_cont.update(self.container_info[key]) env = ior_cmd.get_default_env(str(manager)) manager.assign_hosts(self.hostlist_clients, self.workdir, None) manager.assign_processes(processes) manager.assign_environment(env, True) # run IOR Command try: manager.run() except CommandFailure as _error: results.put("FAIL")
def run_test(self, test_repo, test_name): """ Executable function to be used by test functions below test_repo --location of test repository test_name --name of the test to be run """ # initialize MpioUtils self.mpio = MpioUtils() if not self.mpio.mpich_installed(self.hostlist_clients): self.fail("Exiting Test: Mpich not installed") try: # initialise test specific variables client_processes = self.params.get("np", '/run/client_processes/') # obtaining pool uuid and svc list pool_uuid = self.pool.get_uuid_str() svc_list = "" for i in range(self.createsvc): svc_list += str(int(self.pool.svc.rl_ranks[i])) + ":" svc_list = svc_list[:-1] # running tests self.mpio.run_llnl_mpi4py_hdf5(self.basepath, self.hostfile_clients, pool_uuid, test_repo, test_name, client_processes) # Parsing output to look for failures # stderr directed to stdout stdout = self.logdir + "/stdout" searchfile = open(stdout, "r") error_message = [ "non-zero exit code", "MPI_Abort", "MPI_ABORT", "ERROR" ] for line in searchfile: # pylint: disable=C0200 for i in range(len(error_message)): if error_message[i] in line: self.fail("Test Failed with error_message: {}".format( error_message[i])) except (MpioFailed, DaosApiError) as excep: self.fail("<{0} Test Failed> \n{1}".format(test_name, excep))
def run_test(self, test_repo, test_name): """ Executable function to be used by test functions below test_repo --location of test repository test_name --name of the test to be run """ # Required to run daos command load_mpi("openmpi") # create container self._create_cont() # initialize MpioUtils self.mpio = MpioUtils() if not self.mpio.mpich_installed(self.hostlist_clients): self.fail("Exiting Test: Mpich not installed") # initialize test specific variables client_processes = self.params.get("np", '/run/client_processes/') try: # running tests result = self.mpio.run_mpiio_tests(self.hostfile_clients, self.pool.uuid, test_repo, test_name, client_processes, self.cont_uuid) except MpioFailed as excep: self.fail("<{0} Test Failed> \n{1}".format(test_name, excep)) # Check output for errors error_message = [ "non-zero exit code", "MPI_Abort", "MPI_ABORT", "ERROR" ] for output in (result.stdout, result.stderr): for line in output: for error in error_message: if error in line: self.fail( "Test Failed with error_message: {}".format(error))
def ior_runner_thread(self, results): """Start threads and wait until all threads are finished. Destroy the container at the end of this thread run. Args: results (queue): queue for returning thread results Returns: None """ processes = self.params.get("slots", "/run/ior/clientslots/*") container_info = {} cmd = DaosCommand(os.path.join(self.prefix, "bin")) cmd.set_sub_command("container") cmd.sub_command_class.set_sub_command("destroy") mpio_util = MpioUtils() if mpio_util.mpich_installed(self.hostlist_clients) is False: self.fail("Exiting Test: Mpich not installed") # Iterate through IOR different value and run in sequence for oclass, api, test, flags in product(self.ior_dfs_oclass, self.ior_apis, self.ior_transfer_size, self.ior_flags): # Define the arguments for the ior_runner_thread method ior_cmd = IorCommand() ior_cmd.get_params(self) ior_cmd.set_daos_params(self.server_group, self.pool) ior_cmd.dfs_oclass.update(oclass) ior_cmd.api.update(api) ior_cmd.transfer_size.update(test[0]) ior_cmd.block_size.update(test[1]) ior_cmd.flags.update(flags) container_info["{}{}{}" .format(oclass, api, test[0])] = str(uuid.uuid4()) # Define the job manager for the IOR command manager = Mpirun(ior_cmd, mpitype="mpich") manager.job.dfs_cont.update(container_info ["{}{}{}".format(oclass, api, test[0])]) env = ior_cmd.get_default_env(str(manager)) manager.assign_hosts(self.hostlist_clients, self.workdir, None) manager.assign_processes(processes) manager.assign_environment(env, True) # run IOR Command try: manager.run() except CommandFailure as _error: results.put("FAIL") # Destroy the container created by thread for key in container_info: cmd.sub_command_class.sub_command_class.pool.value = self.pool.uuid cmd.sub_command_class.sub_command_class.svc.value = \ self.pool.svc_ranks cmd.sub_command_class.sub_command_class.cont.value = \ container_info[key] try: cmd._get_result() except CommandFailure as _error: results.put("FAIL")