def execution(self, agg_trigger=False): """ Common method to run the tests. Args: agg_trigger(Boolean): Check if Aggregation is happening after IO or not, By default it's False. """ self.pool.connect() self.log.info("pool_percentage Before = %s", self.pool.pool_percentage_used()) # Write the IOR data set with given all the EC object type self.ior_write_dataset() if agg_trigger: # Verify that Aggregation is getting started if not any(check_aggregation_status(self.pool).values()): self.fail("Aggregation failed to start..") else: # Verify that Aggregation is not starting if any(check_aggregation_status(self.pool).values()) is True: self.fail("Aggregation should not happens...") # Read IOR data and verify content self.ior_read_dataset() # Kill the last server rank self.server_managers[0].stop_ranks([self.server_count - 1], self.d_log, force=True) # Wait for rebuild to complete self.pool.wait_for_rebuild(False) # Read IOR data and verify for different EC object data still OK # written before killing the single server self.ior_read_dataset() # Kill the another server rank self.server_managers[0].stop_ranks([self.server_count - 2], self.d_log, force=True) # Wait for rebuild to complete self.pool.wait_for_rebuild(False) # Read IOR data and verify for different EC object and different sizes # written before killing the second server. # Only +2 (Parity) data will be intact so read and verify only +2 IOR # data set self.ior_read_dataset(parity=2)
def execution(self, agg_check=None): """ Common test execution method to write data, verify aggregation, restart all the servers and read data. Args: agg_check: When to check Aggregation status.Either before restarting all the servers or after.Default is None so not to check aggregation, but wait for 20 seconds and restart the servers. """ # Write all EC object data to NVMe self.ior_write_dataset(operation="Auto_Write", percent=self.percent) self.log.info(self.pool.pool_percentage_used()) # Write all EC object data to SCM self.ior_write_dataset(storage='SCM', operation="Auto_Write", percent=self.percent) self.log.info(self.pool.pool_percentage_used()) if not agg_check: # Set time mode aggregation self.pool.set_property("reclaim", "time") # Aggregation will start in 20 seconds after it sets to time mode. So wait for 20 # seconds and restart all the servers. time.sleep(20) if agg_check == "Before": # Verify if Aggregation is getting started if not any( check_aggregation_status(self.pool, attempt=50).values()): self.fail( "Aggregation failed to start Before server restart..") # Shutdown the servers and restart self.get_dmg_command().system_stop(True) time.sleep(5) self.get_dmg_command().system_start() if agg_check == "After": # Verify if Aggregation is getting started if not any( check_aggregation_status(self.pool, attempt=50).values()): self.fail("Aggregation failed to start After server restart..") # Read all EC object data from NVMe self.ior_read_dataset(operation="Auto_Read", percent=self.percent) # Read all EC object data which was written on SCM self.read_set_from_beginning = False self.ior_read_dataset(storage='SCM', operation="Auto_Read", percent=self.percent)
def test_ec_aggregation_time(self): """Jira ID: DAOS-7325. Test Description: Test Erasure code object aggregation time mode with IOR. Use Case: Create the pool,Set aggregation as time mode. run IOR with supported EC object type classes. Verify the Aggregation gets triggered in parallel and space is getting reclaimed. Verify the IOR read data at the end. :avocado: tags=all,full_regression :avocado: tags=hw,large,ib2 :avocado: tags=ec,aggregation,ec_array,ec_aggregation :avocado: tags=ec_aggregation_time """ # Set time mode aggregation self.pool.set_property("reclaim", "time") self.pool.connect() self.log.info("pool_percentage Before = %s ", self.pool.pool_percentage_used()) # Write the IOR data set with given all the EC object type self.ior_write_dataset() # Verify if Aggregation is getting started if not any(check_aggregation_status(self.pool).values()): self.fail("Aggregation failed to start..") # Read IOR data and verify content self.ior_read_dataset()
def test_ec_aggregation_default(self): """Jira ID: DAOS-7325. Test Description: Test Erasure code object aggregation enabled(default) mode with IOR. Use Case: Create the pool, run IOR with supported EC object type classes. Verify the Aggregation gets triggered and space is getting reclaimed. Verify the IOR read data at the end. :avocado: tags=all,full_regression :avocado: tags=hw,large,ib2 :avocado: tags=ec,aggregation,ec_array,ec_aggregation :avocado: tags=ec_aggregation_default """ self.pool.connect() self.log.info("pool_percentage Before = %s ", self.pool.pool_percentage_used()) for oclass in self.obj_class: for sizes in self.ior_chu_trs_blk_size: # Write single IOR self.ior_write_single_dataset(oclass, sizes) # Verify if Aggregation is getting started if not any(check_aggregation_status(self.pool).values()): self.fail("Aggregation failed to start..") # Read single IOR self.ior_read_single_dataset(oclass, sizes) # Increase the container UUID count and read the latest data. self.cont_number += 1
def test_ec_aggregation_disabled(self): """Jira ID: DAOS-7325. Test Description: Test Erasure code object aggregation disabled mode with IOR. Use Case: Create the pool, disabled aggregation, run IOR with supported EC object type. Verify that Aggregation should not triggered. Verify the IOR read data at the end. :avocado: tags=all,full_regression :avocado: tags=hw,large,ib2 :avocado: tags=ec,aggregation,ec_array,ec_aggregation :avocado: tags=ec_aggregation_disabled """ # Disable the aggregation self.pool.set_property("reclaim", "disabled") self.pool.connect() self.log.info("pool_percentage Before = %s ", self.pool.pool_percentage_used()) # Write the IOR data set with given all the EC object type self.ior_write_dataset() # Verify if Aggregation is getting started if any(check_aggregation_status(self.pool).values()) is True: self.fail("Aggregation should not happens...") # Read IOR data and verify content self.ior_read_dataset()
def test_ec_degrade(self): """Jira ID: DAOS-5893. Test Description: Test Erasure code object with IOR. Use Case: Create the pool, disabled rebuild, run IOR with supported EC object type class for small and large transfer sizes. kill single server, verify all IOR read data and verified. :avocado: tags=all,full_regression :avocado: tags=hw,large,ib2 :avocado: tags=ec,ec_array,ec_disabled_rebuild,rebuild :avocado: tags=ec_disabled_rebuild_array """ # Disabled pool Rebuild self.pool.set_property("self_heal", "exclude") # self.pool.set_property("reclaim", "disabled") # Write the IOR data set with given all the EC object type self.ior_write_dataset() # Verify if Aggregation is getting started if not any(check_aggregation_status(self.pool).values()): self.fail("Aggregation failed to start..") # Kill the last server rank and wait for 20 seconds, Rebuild is disabled # so data should not be rebuild self.server_managers[0].stop_ranks([self.server_count - 1], self.d_log, force=True) time.sleep(20) # Read IOR data and verify for different EC object and different sizes # written before killing the single server self.ior_read_dataset() # Kill the another server rank and wait for 20 seconds,Rebuild will # not happens because i's disabled.Read/verify data with Parity 2. self.server_managers[0].stop_ranks([self.server_count - 2], self.d_log, force=True) time.sleep(20) # Read IOR data and verify for different EC object and different sizes # written before killing the single server self.ior_read_dataset(parity=2)
def execution(self, rebuild_mode): """ Test execution Args: rebuild_mode: On-line or off-line rebuild mode """ # Kill last server rank first self.rank_to_kill = self.server_count - 1 if 'on-line' in rebuild_mode: # Enabled on-line rebuild for the test self.set_online_rebuild = True # Write the Fio data and kill server if rebuild_mode is on-line self.start_online_fio() # Verify Aggregation should start for Partial stripes IO if not any(check_aggregation_status(self.pool, attempt=60).values()): self.fail("Aggregation failed to start..") if 'off-line' in rebuild_mode: self.server_managers[0].stop_ranks( [self.server_count - 1], self.d_log, force=True) # Adding unlink option for final read command if int(self.container.properties.value.split(":")[1]) == 1: self.fio_cmd._jobs['test'].unlink.value = 1 # Read and verify the original data. self.fio_cmd._jobs['test'].rw.value = self.read_option self.fio_cmd.run() # If RF is 2 kill one more server and validate the data is not corrupted. if int(self.container.properties.value.split(":")[1]) == 2: self.fio_cmd._jobs['test'].unlink.value = 1 self.log.info("RF is 2,So kill another server and verify data") # Kill one more server rank self.server_managers[0].stop_ranks([self.server_count - 2], self.d_log, force=True) # Read and verify the original data. self.fio_cmd.run()
def test_ec_offline_agg_during_rebuild(self): """Jira ID: DAOS-7313. Test Description: Test Erasure code object aggregation time mode with IOR. Use Case: Create the pool, disabled aggregation. run IOR with supported EC object type with partial strip. Enable Aggregation as time mode. Wait for 20 seconds where it will trigger the aggregation Kill single server after 20 seconds and wait for rebuild. Read and verify all the data. Kill second server and wait for rebuild. Read and verify data with +2 Parity with no data corruption. :avocado: tags=all,full_regression :avocado: tags=hw,large,ib2 :avocado: tags=ec,aggregation,ec_array,ec_aggregation :avocado: tags=ec_offline_agg_during_rebuild """ # Disable the aggregation self.pool.set_property("reclaim", "disabled") self.pool.connect() # Write the IOR data set with given all the EC object type self.ior_write_dataset() # Read IOR data and verify content self.ior_read_dataset() # Set time mode aggregation self.pool.set_property("reclaim", "time") # Aggregation will start in 20 seconds after it sets to time mode. # So wait for 20 seconds and kill the last server rank time.sleep(20) self.server_managers[0].stop_ranks([self.server_count - 1], self.d_log, force=True) # Verify if Aggregation is getting started if not any(check_aggregation_status(self.pool).values()): self.fail("Aggregation failed to start..") # Wait for rebuild to complete self.pool.wait_for_rebuild(False) # Read IOR data and verify for different EC object data still OK # written before killing the single server self.ior_read_dataset() # Kill the another server rank self.server_managers[0].stop_ranks([self.server_count - 2], self.d_log, force=True) # Wait for rebuild to complete self.pool.wait_for_rebuild(False) # Read IOR data and verify for different EC object and different sizes # written before killing the second server. # Only +2 (Parity) data will be intact so read and verify only +2 IOR # data set self.ior_read_dataset(parity=2)