class ParallelIo(FioBase): """Base Parallel IO test class. :avocado: recursive """ def __init__(self, *args, **kwargs): """Initialize a ParallelIo object.""" super(ParallelIo, self).__init__(*args, **kwargs) self.dfuse = None self.cont_count = None self.container = [] def setUp(self): """Set up each test case.""" # Start the servers and agents super(ParallelIo, self).setUp() def tearDown(self): """Tear down each test case.""" try: if self.dfuse: self.dfuse.stop() finally: # Stop the servers and agents super(ParallelIo, self).tearDown() def create_pool(self): """Create a TestPool object to use with ior.""" # Get the pool params self.pool = TestPool( self.context, dmg_command=self.get_dmg_command()) self.pool.get_params(self) # Create a pool self.pool.create() def create_cont(self): """Create a TestContainer object to be used to create container.""" # Get container params container = TestContainer( self.pool, daos_command=DaosCommand(self.bin)) container.get_params(self) # create container container.create() self.container.append(container) def start_dfuse(self): """Create a DfuseCommand object to start dfuse. """ # Get Dfuse params self.dfuse = Dfuse(self.hostlist_clients, self.tmp) self.dfuse.get_params(self) # update dfuse params self.dfuse.set_dfuse_params(self.pool) self.dfuse.set_dfuse_exports(self.server_managers[0], self.client_log) try: # start dfuse self.dfuse.run() except CommandFailure as error: self.log.error("Dfuse command %s failed on hosts %s", str(self.dfuse), self.dfuse.hosts, exc_info=error) self.fail("Test was expected to pass but it failed.\n") def test_parallelio(self): """Jira ID: DAOS-3775. Test Description: Purpose of this test is to mount dfuse and verify multiple containers using fio. Use cases: Mount dfuse using pool uuid. Create multiple containers under that dfuse mount point. Check those containers are accessible from that mount point. Perform io to those containers using FIO Delete one of the containers Check if dfuse is still running. If not, fail the test and exit. Otherwise, try accessing the deleted container. This should fail. Check dfuse again. :avocado: tags=all,hw,daosio,medium,ib2,full_regression,parallelio """ # get test params for cont and pool count self.cont_count = self.params.get("cont_count", '/run/container/*') threads = [] # Create a pool and start dfuse. self.create_pool() self.start_dfuse() # create multiple containers in parallel cont_threads = [] for _ in range(self.cont_count): cont_thread = threading.Thread(target=self.create_cont()) cont_threads.append(cont_thread) # start container create job for cont_job in cont_threads: cont_job.start() # wait for container create to finish for cont_job in cont_threads: cont_job.join() # check if all the created containers can be accessed and perform # io on each container using fio in parallel for _, cont in enumerate(self.container): dfuse_cont_dir = self.dfuse.mount_dir.value + "/" + cont.uuid cmd = u"ls -a {}".format(dfuse_cont_dir) try: # execute bash cmds ret_code = general_utils.pcmd( self.hostlist_clients, cmd, timeout=30) if 0 not in ret_code: error_hosts = NodeSet( ",".join( [str(node_set) for code, node_set in ret_code.items() if code != 0])) raise CommandFailure( "Error running '{}' on the following " "hosts: {}".format(cmd, error_hosts)) # report error if any command fails except CommandFailure as error: self.log.error("ParallelIo Test Failed: %s", str(error)) self.fail("Test was expected to pass but " "it failed.\n") # run fio on all containers thread = threading.Thread(target=self.execute_fio, args=( self.dfuse.mount_dir.value + "/" + cont.uuid, False)) threads.append(thread) thread.start() # wait for all fio jobs to be finished for job in threads: job.join() # destroy first container container_to_destroy = self.container[0].uuid self.container[0].destroy(1) # check dfuse if it is running fine self.dfuse.check_running() # try accessing destroyed container, it should fail try: self.execute_fio(self.dfuse.mount_dir.value + "/" + \ container_to_destroy, False) self.fail("Fio was able to access destroyed container: {}".\ format(self.container[0].uuid)) except CommandFailure as error: self.log.info("This run is expected to fail") # check dfuse is still running after attempting to access deleted # container. self.dfuse.check_running()
class ParallelIo(FioBase, IorTestBase): """Base Parallel IO test class. :avocado: recursive """ def __init__(self, *args, **kwargs): """Initialize a ParallelIo object.""" super(ParallelIo, self).__init__(*args, **kwargs) self.dfuse = None self.cont_count = None self.pool_count = None self.statvfs_info_initial = None self.statvfs_before_cont_destroy = None self.statvfs_after_cont_destroy = None self.pool = [] self.container = [] def setUp(self): """Set up each test case.""" # Start the servers and agents super(ParallelIo, self).setUp() def tearDown(self): """Tear down each test case.""" try: if self.dfuse: self.dfuse.stop() finally: # Stop the servers and agents super(ParallelIo, self).tearDown() def create_pool(self): """Create a TestPool object to use with ior.""" # Get the pool params pool = TestPool(self.context, dmg_command=self.get_dmg_command()) pool.get_params(self) # Create a pool pool.create() self.pool.append(pool) # pylint: disable=arguments-differ def create_cont(self, pool): """Create a TestContainer object to be used to create container. Args: pool (TestPool): TestPool object type for which container needs to be created """ # Get container params container = TestContainer(pool, daos_command=DaosCommand(self.bin)) container.get_params(self) # create container container.create() self.container.append(container) def start_dfuse(self, pool=None): """Create a DfuseCommand object to start dfuse. Args: pool (TestPool): Test pool object if dfuse is intended to be started using pool uuid option. """ # Get Dfuse params self.dfuse = Dfuse(self.hostlist_clients, self.tmp) self.dfuse.get_params(self) # update dfuse params if pool: self.dfuse.set_dfuse_params(pool) self.dfuse.set_dfuse_exports(self.server_managers[0], self.client_log) try: # start dfuse self.dfuse.run() except CommandFailure as error: self.log.error("Dfuse command %s failed on hosts %s", str(self.dfuse), self.dfuse.hosts, exc_info=error) self.fail("Test was expected to pass but it failed.\n") def stat_bfree(self, path): """Get stat bfree Args: path (str): path to get free block size of. Returns: integer value of stat free blocks """ cmd = [ "ssh", "{}@{}".format(getuser(), self.hostlist_clients[0]), "stat -c%a -f {}".format(path) ] try: result = subprocess.check_output(cmd) except subprocess.CalledProcessError as err: self.fail("Get free block size method failed with: {}".format(err)) return int(result) def statvfs_pool(self, path): """Method to obtain free space using statvfs Args: path (str): path for which free space needs to be obtained for. Returns: List containing free space info for each pool supplied in pool_obj. """ statvfs_list = [] for _, pool in enumerate(self.pool): dfuse_pool_dir = str(path + "/" + pool.uuid) statvfs_info = self.stat_bfree(dfuse_pool_dir) statvfs_list.append(statvfs_info) self.log.info("Statvfs List Output: %s", statvfs_list) return statvfs_list def verify_aggregation(self, reduced_space, count): """Verify if expected space is returned for each pool after containers were destroyed. If not, wait for 60 secs and check again. Wait 4 times, otherwise exit the test with a failure. Args: reduced_space: expected space to be returned """ counter = 1 while (self.statvfs_after_cont_destroy[count] < self.statvfs_before_cont_destroy[count] + reduced_space): # try to wait for 4 x 60 secs for aggregation to be completed # or else exit the test with a failure. if counter > 4: self.log.info("Free space before io: %s", self.statvfs_info_initial) self.log.info("Free space after io: %s", self.statvfs_before_cont_destroy) self.log.info("Free space at test termination: %s", self.statvfs_after_cont_destroy) self.fail("Aggregation did not complete as expected") time.sleep(60) self.statvfs_after_cont_destroy = self.statvfs_pool( self.dfuse.mount_dir.value) counter += 1 def test_parallelio(self): """Jira ID: DAOS-3775. Test Description: Purpose of this test is to mount dfuse and verify multiple containers using fio. Use cases: Mount dfuse using pool uuid. Create multiple containers under that dfuse mount point. Check those containers are accessible from that mount point. Perform io to those containers using FIO Delete one of the containers Check if dfuse is still running. If not, fail the test and exit. Otherwise, try accessing the deleted container. This should fail. Check dfuse again. :avocado: tags=all,hw,daosio,medium,ib2,full_regression,parallelio """ # get test params for cont and pool count self.cont_count = self.params.get("cont_count", '/run/container/*') threads = [] # Create a pool and start dfuse. self.create_pool() self.start_dfuse(self.pool[0]) # create multiple containers for _ in range(self.cont_count): self.create_cont(self.pool[0]) # check if all the created containers can be accessed and perform # io on each container using fio in parallel for _, cont in enumerate(self.container): dfuse_cont_dir = self.dfuse.mount_dir.value + "/" + cont.uuid cmd = u"ls -a {}".format(dfuse_cont_dir) try: # execute bash cmds ret_code = general_utils.pcmd(self.hostlist_clients, cmd, timeout=30) if 0 not in ret_code: error_hosts = NodeSet(",".join([ str(node_set) for code, node_set in ret_code.items() if code != 0 ])) raise CommandFailure("Error running '{}' on the following " "hosts: {}".format(cmd, error_hosts)) # report error if any command fails except CommandFailure as error: self.log.error("ParallelIo Test Failed: %s", str(error)) self.fail("Test was expected to pass but " "it failed.\n") # run fio on all containers thread = threading.Thread(target=self.execute_fio, args=(self.dfuse.mount_dir.value + "/" + cont.uuid, False)) threads.append(thread) thread.start() # wait for all fio jobs to be finished for job in threads: job.join() # destroy first container container_to_destroy = self.container[0].uuid self.container[0].destroy(1) # check dfuse if it is running fine self.dfuse.check_running() # try accessing destroyed container, it should fail try: self.execute_fio(self.dfuse.mount_dir.value + "/" + \ container_to_destroy, False) self.fail("Fio was able to access destroyed container: {}".\ format(self.container[0].uuid)) except CommandFailure as error: self.log.info("This run is expected to fail") # check dfuse is still running after attempting to access deleted # container. self.dfuse.check_running() def test_multipool_parallelio(self): """Jira ID: DAOS-3775. Test Description: Purpose of this test is to verify aggregation across multiple pools and containers. Use cases: Create 10 pools Create 10 containers under each pool. Record statvfs free space for each pool. Perform parallel io to each pool without deleting the file after write. Record free space using statvfs after write. Delete half of the containers from each pool. Calculate the expected amount of data to be deleted when containers are destroyed. Record free space after container destroy. Loop until either the all space is returned back after aggregation completion or exit the loop after trying for 240 secs of wait and fail the test. :avocado: tags=all,hw,daosio,medium,ib2,full_regression :avocado: tags=multipoolparallelio """ # test params threads = [] pool_threads = [] cont_threads = [] self.pool_count = self.params.get("pool_count", '/run/pool/*') self.cont_count = self.params.get("cont_count", '/run/container/*') processes = self.params.get("np", '/run/ior/client_processes/*') # Create pools in parallel. for _ in range(self.pool_count): pool_thread = threading.Thread(target=self.create_pool()) pool_threads.append(pool_thread) pool_thread.start() # wait for container create to finish for pool_job in pool_threads: pool_job.join() # start dfuse using --svc option only. self.start_dfuse() # record free space using statvfs before any data is written. self.statvfs_info_initial = self.statvfs_pool( self.dfuse.mount_dir.value) # Create 10 containers for each pool. Container create process cannot # be parallelised as different container create could complete at # different times and get appended in the self.container variable in # unorderly manner, causing problems during the write process. for _, pool in enumerate(self.pool): for _ in range(self.cont_count): self.create_cont(pool) # Try to access each dfuse mounted container using ls. Once it is # accessed successfully, go ahead and perform io on that location # using ior. This process of performing io is done in parallel for # all containers using threads. for pool_count, pool in enumerate(self.pool): dfuse_pool_dir = str(self.dfuse.mount_dir.value + "/" + pool.uuid) for counter in range(self.cont_count): cont_num = (pool_count * self.cont_count) + counter dfuse_cont_dir = str(dfuse_pool_dir + "/" + self.container[cont_num].uuid) cmd = u"###ls -a {}".format(dfuse_cont_dir) self.execute_cmd(cmd) # run ior on all containers test_file = dfuse_cont_dir + "/testfile" self.ior_cmd.test_file.update(test_file) self.ior_cmd.set_daos_params(self.server_group, pool, self.container[cont_num].uuid) thread = threading.Thread( target=self.run_ior, args=(self.get_ior_job_manager_command(), processes, None, False)) threads.append(thread) thread.start() # wait for all ior jobs to be finished for job in threads: job.join() # Record free space after io self.statvfs_before_cont_destroy = self.statvfs_pool( self.dfuse.mount_dir.value) # Destroy half of the containers from each pool pfinal = 0 for count in range(self.cont_count): pinitial = pfinal pfinal = pinitial + (self.cont_count // 2) del self.container[pinitial:pfinal] for cont in self.container: cont_thread = threading.Thread(target=cont.destroy) cont_threads.append(cont_thread) cont_thread.start() for destroy_job in cont_threads: destroy_job.join() # Record free space after container destroy. self.statvfs_after_cont_destroy = self.statvfs_pool( self.dfuse.mount_dir.value) # Calculate the expected space to be returned after containers # are destroyed. reduced_space = (self.cont_count * int(self.ior_cmd.block_size.value)) / 2 # Verify if expected space is returned for each pool after containers # were destroyed. If not, wait for 60 secs and check again. Wait 4 # times, otherwise exit the test with a failure. for count in range(self.pool_count): thread = threading.Thread(target=self.verify_aggregation, args=(reduced_space, count)) threads.append(thread) thread.start() for job in threads: job.join()
class DfuseContainerCheck(TestWithServers): """Base Dfuse Container check test class. :avocado: recursive """ def __init__(self, *args, **kwargs): """Initialize a DfuseContainerCheck object.""" super(DfuseContainerCheck, self).__init__(*args, **kwargs) self.dfuse = None self.pool = None self.container = None def setUp(self): """Set up each test case.""" # Start the servers and agents super(DfuseContainerCheck, self).setUp() def tearDown(self): """Tear down each test case.""" try: if self.dfuse: self.dfuse.stop() finally: # Stop the servers and agents super(DfuseContainerCheck, self).tearDown() def create_pool(self): """Create a TestPool object to use with ior.""" # Get the pool params self.pool = TestPool(self.context, dmg_command=self.get_dmg_command()) self.pool.get_params(self) # Create a pool self.pool.create() def start_dfuse(self): """Create a DfuseCommand object to start dfuse. """ # Get Dfuse params self.dfuse = Dfuse(self.hostlist_clients, self.tmp) self.dfuse.get_params(self) # update dfuse params self.dfuse.set_dfuse_params(self.pool) self.dfuse.set_dfuse_cont_param(self.container) self.dfuse.set_dfuse_exports(self.server_managers[0], self.client_log) try: # start dfuse self.dfuse.run(False) except CommandFailure as error: self.log.error("Dfuse command %s failed on hosts %s", str(self.dfuse), self.dfuse.hosts, exc_info=error) self.fail("Test was expected to pass but it failed.\n") def test_dfusecontainercheck(self): """Jira ID: DAOS-3635. Test Description: Purpose of this test is to try and mount different container types to dfuse and check the behavior. Use cases: Create pool Create container of type default Try to mount to dfuse and check the behaviour. Create container of type POSIX. Try to mount to dfuse and check the behaviour. :avocado: tags=all,small,full_regression,dfusecontainercheck """ # get test params for cont and pool count cont_types = self.params.get("cont_types", '/run/container/*') # Create a pool and start dfuse. self.create_pool() for cont_type in cont_types: # Get container params self.container = TestContainer(self.pool, daos_command=DaosCommand(self.bin)) self.container.get_params(self) # create container if cont_type == "POSIX": self.container.type.update(cont_type) self.container.create() try: # mount fuse self.start_dfuse() # check if fuse got mounted self.dfuse.check_running() # fail the test if fuse mounts with non-posix type container if cont_type == "": self.fail( "Non-Posix type container got mounted over dfuse") except CommandFailure as error: # expected to throw CommandFailure exception for non-posix type # container if cont_type == "": self.log.info( "Expected behaviour: Default container type \ is expected to fail on dfuse mount: %s", str(error)) # fail the test if exception is caught for POSIX type container elif cont_type == "POSIX": self.log.error( "Posix Container dfuse mount \ failed: %s", str(error)) self.fail("Posix container type was expected to mount \ over dfuse") # stop fuse and container for next iteration if not cont_type == "": self.dfuse.stop() self.container.destroy(1)