def new_cleaning_parameters_random_values(cleaning_policy): if cleaning_policy == CleaningPolicy.alru: alru_params_range = FlushParametersAlru().alru_params_range() wake_up_time_random_value = Time(seconds=random.randint( *alru_params_range.wake_up_time)) staleness_time_random_value = Time(seconds=random.randint( *alru_params_range.staleness_time)) flush_max_buffers_random_value = random.randint( *alru_params_range.flush_max_buffers) activity_threshold_random_value = Time(milliseconds=random.randint( *alru_params_range.activity_threshold)) cleaning_params = FlushParametersAlru() cleaning_params.wake_up_time = wake_up_time_random_value cleaning_params.staleness_time = staleness_time_random_value cleaning_params.flush_max_buffers = flush_max_buffers_random_value cleaning_params.activity_threshold = activity_threshold_random_value if cleaning_policy == CleaningPolicy.acp: acp_params_range = FlushParametersAcp().acp_params_range() wake_up_time_random_value = Time(milliseconds=random.randint( *acp_params_range.wake_up_time)) flush_max_buffers_random_value = random.randint( *acp_params_range.flush_max_buffers) cleaning_params = FlushParametersAcp() cleaning_params.wake_up_time = wake_up_time_random_value cleaning_params.flush_max_buffers = flush_max_buffers_random_value return cleaning_params
def set_cleaning_policy_and_params(cache, cleaning_policy): if cleaning_policy != CleaningPolicy.DEFAULT: cache.set_cleaning_policy(cleaning_policy) current_cleaning_policy = cache.get_cleaning_policy() if current_cleaning_policy != cleaning_policy: TestRun.LOGGER.error(f"Cleaning policy is {current_cleaning_policy}, " f"should be {cleaning_policy}") if cleaning_policy == CleaningPolicy.alru: alru_params = FlushParametersAlru() alru_params.wake_up_time = Time(seconds=10) alru_params.staleness_time = Time(seconds=2) alru_params.flush_max_buffers = 100 alru_params.activity_threshold = Time(milliseconds=1000) cache.set_params_alru(alru_params) current_alru_params = cache.get_flush_parameters_alru() if current_alru_params != alru_params: failed_params = "" if current_alru_params.wake_up_time != alru_params.wake_up_time: failed_params += ( f"Wake Up time is {current_alru_params.wake_up_time}, " f"should be {alru_params.wake_up_time}\n") if current_alru_params.staleness_time != alru_params.staleness_time: failed_params += ( f"Staleness Time is {current_alru_params.staleness_time}, " f"should be {alru_params.staleness_time}\n") if current_alru_params.flush_max_buffers != alru_params.flush_max_buffers: failed_params += ( f"Flush Max Buffers is {current_alru_params.flush_max_buffers}, " f"should be {alru_params.flush_max_buffers}\n") if current_alru_params.activity_threshold != alru_params.activity_threshold: failed_params += ( f"Activity Threshold is {current_alru_params.activity_threshold}, " f"should be {alru_params.activity_threshold}\n") TestRun.LOGGER.error( f"ALRU parameters did not switch properly:\n{failed_params}") if cleaning_policy == CleaningPolicy.acp: acp_params = FlushParametersAcp() acp_params.wake_up_time = Time(milliseconds=100) acp_params.flush_max_buffers = 64 cache.set_params_acp(acp_params) current_acp_params = cache.get_flush_parameters_acp() if current_acp_params != acp_params: failed_params = "" if current_acp_params.wake_up_time != acp_params.wake_up_time: failed_params += ( f"Wake Up time is {current_acp_params.wake_up_time}, " f"should be {acp_params.wake_up_time}\n") if current_acp_params.flush_max_buffers != acp_params.flush_max_buffers: failed_params += ( f"Flush Max Buffers is {current_acp_params.flush_max_buffers}, " f"should be {acp_params.flush_max_buffers}\n") TestRun.LOGGER.error( f"ACP parameters did not switch properly:\n{failed_params}")
def test_acp_param_wake_up_time(cache_line_size, cache_mode): """ title: Functional test for ACP wake-up parameter. description: | Verify if interval between ACP cleaning iterations is not longer than wake-up time parameter value. pass_criteria: - ACP flush iterations are triggered with defined frequency. """ with TestRun.step("Test prepare."): error_threshold_ms = 50 generated_vals = get_random_list( min_val=FlushParametersAcp.acp_params_range().wake_up_time[0], max_val=FlushParametersAcp.acp_params_range().wake_up_time[1], n=10, ) acp_configs = [] for config in generated_vals: acp_configs.append( FlushParametersAcp(wake_up_time=Time(milliseconds=config))) acp_configs.append(FlushParametersAcp.default_acp_params()) with TestRun.step("Prepare partitions."): core_size = Size(5, Unit.GibiByte) cache_device = TestRun.disks["cache"] core_device = TestRun.disks["core"] cache_device.create_partitions([Size(10, Unit.GibiByte)]) core_device.create_partitions([core_size]) with TestRun.step( f"Start cache in {cache_mode} with {cache_line_size} and add core." ): cache = casadm.start_cache(cache_device.partitions[0], cache_mode, cache_line_size) core = cache.add_core(core_device.partitions[0]) with TestRun.step("Set cleaning policy to NOP."): cache.set_cleaning_policy(CleaningPolicy.nop) with TestRun.step("Start IO in background."): fio = get_fio_cmd(core, core_size) fio.run_in_background() time.sleep(10) with TestRun.step("Set cleaning policy to ACP."): cache.set_cleaning_policy(CleaningPolicy.acp) with TestRun.group("Verify IO number for different wake_up_time values."): for acp_config in acp_configs: with TestRun.step(f"Setting {acp_config}"): cache.set_params_acp(acp_config) accepted_interval_threshold = ( acp_config.wake_up_time.total_milliseconds() + error_threshold_ms) with TestRun.step( "Using blktrace verify if interval between ACP cleaning iterations " f"is shorter or equal than wake-up parameter value " f"(including {error_threshold_ms}ms error threshold)"): blktrace = BlkTrace(core.core_device, BlkTraceMask.write) blktrace.start_monitoring() time.sleep(15) blktrace_output = blktrace.stop_monitoring() for (prev, curr) in zip(blktrace_output, blktrace_output[1:]): if not new_acp_iteration(prev, curr): continue interval_ms = (curr.timestamp - prev.timestamp) / 10**6 if interval_ms > accepted_interval_threshold: TestRun.LOGGER.error( f"{interval_ms} is not within accepted range for " f"{acp_config.wake_up_time.total_milliseconds()} " f"wake_up_time param value.") with TestRun.step("Stop all caches"): kill_all_io() casadm.stop_all_caches()
def test_acp_param_flush_max_buffers(cache_line_size, cache_mode): """ title: Functional test for ACP flush-max-buffers parameter. description: | Verify if there is appropriate number of I/O requests between wake-up time intervals, which depends on flush-max-buffer parameter. pass_criteria: - ACP triggered dirty data flush - Number of writes to core is lower or equal than flush_max_buffers """ with TestRun.step("Test prepare."): buffer_values = get_random_list( min_val=FlushParametersAcp.acp_params_range().flush_max_buffers[0], max_val=FlushParametersAcp.acp_params_range().flush_max_buffers[1], n=10, ) default_config = FlushParametersAcp.default_acp_params() acp_configs = [ FlushParametersAcp(flush_max_buffers=buf, wake_up_time=Time(seconds=1)) for buf in buffer_values ] acp_configs.append(default_config) with TestRun.step("Prepare partitions."): core_size = Size(5, Unit.GibiByte) cache_device = TestRun.disks["cache"] core_device = TestRun.disks["core"] cache_device.create_partitions([Size(10, Unit.GibiByte)]) core_device.create_partitions([core_size]) with TestRun.step( f"Start cache in {cache_mode} with {cache_line_size} and add core." ): cache = casadm.start_cache(cache_device.partitions[0], cache_mode, cache_line_size) core = cache.add_core(core_device.partitions[0]) with TestRun.step("Set cleaning policy to NOP."): cache.set_cleaning_policy(CleaningPolicy.nop) with TestRun.step("Start IO in background."): fio = get_fio_cmd(core, core_size) fio.run_in_background() time.sleep(10) with TestRun.step("Set cleaning policy to ACP."): cache.set_cleaning_policy(CleaningPolicy.acp) with TestRun.group( "Verify IO number for different max_flush_buffers values."): for acp_config in acp_configs: with TestRun.step(f"Setting {acp_config}"): cache.set_params_acp(acp_config) with TestRun.step( "Using blktrace verify if there is appropriate number of I/O requests, " "which depends on flush-max-buffer parameter."): blktrace = BlkTrace(core.core_device, BlkTraceMask.write) blktrace.start_monitoring() time.sleep(20) blktrace_output = blktrace.stop_monitoring() cleaning_started = False flush_writes = 0 for (prev, curr) in zip(blktrace_output, blktrace_output[1:]): if cleaning_started and write_to_core(prev): flush_writes += 1 if new_acp_iteration(prev, curr): if cleaning_started: if flush_writes <= acp_config.flush_max_buffers: flush_writes = 0 else: TestRun.LOGGER.error( f"Incorrect number of handled io requests. " f"Expected {acp_config.flush_max_buffers} - " f"actual {flush_writes}") flush_writes = 0 cleaning_started = True if not cleaning_started: TestRun.fail(f"ACP flush not triggered for {acp_config}") with TestRun.step("Stop all caches"): kill_all_io() casadm.stop_all_caches()
def test_set_get_cleaning_params(cache_mode, cleaning_policy): """ title: Test for setting and reading cleaning parameters. description: | Verify that it is possible to set and read all available cleaning parameters for all cleaning policies using casadm --set-param and --get-param options. pass_criteria: - All cleaning parameters are set to given values. - All cleaning parameters displays proper values. """ with TestRun.step("Partition cache and core devices"): cache_dev, core_dev = storage_prepare() with TestRun.step( f"Start {caches_count} caches in {cache_mode} cache mode " f"and add {cores_per_cache} cores per cache"): caches, cores = cache_prepare(cache_mode, cache_dev, core_dev) with TestRun.step(f"Set cleaning policy to {cleaning_policy}"): if cleaning_policy != CleaningPolicy.DEFAULT: for i in range(caches_count): caches[i].set_cleaning_policy(cleaning_policy) for i in range(caches_count): current_cleaning_policy = caches[i].get_cleaning_policy() if current_cleaning_policy != cleaning_policy: TestRun.fail( f"Cleaning policy for cache nr {caches[i].cache_id} " f"is {current_cleaning_policy}, should be {cleaning_policy}" ) with TestRun.step( f"Check {cleaning_policy} cleaning policy default parameters"): if cleaning_policy == CleaningPolicy.alru: default_cleaning_params = FlushParametersAlru.default_alru_params() if cleaning_policy == CleaningPolicy.acp: default_cleaning_params = FlushParametersAcp.default_acp_params() for i in range(caches_count): check_cleaning_parameters(caches[i], cleaning_policy, default_cleaning_params) with TestRun.step(f"Set new random values for {cleaning_policy} " f"cleaning policy parameters for one cache instance"): for check in range(number_of_checks): random_cleaning_params = new_cleaning_parameters_random_values( cleaning_policy) if cleaning_policy == CleaningPolicy.alru: caches[0].set_params_alru(random_cleaning_params) if cleaning_policy == CleaningPolicy.acp: caches[0].set_params_acp(random_cleaning_params) # Check changed parameters for first cache instance: check_cleaning_parameters(caches[0], cleaning_policy, random_cleaning_params) # Check default parameters for other cache instances: for i in range(1, caches_count): check_cleaning_parameters(caches[i], cleaning_policy, default_cleaning_params) with TestRun.step(f"Set new random values for {cleaning_policy} " f"cleaning policy parameters for all cache instances"): for check in range(number_of_checks): cleaning_params = [] for i in range(caches_count): random_cleaning_params = new_cleaning_parameters_random_values( cleaning_policy) cleaning_params.append(random_cleaning_params) if cleaning_policy == CleaningPolicy.alru: caches[i].set_params_alru(random_cleaning_params) if cleaning_policy == CleaningPolicy.acp: caches[i].set_params_acp(random_cleaning_params) for i in range(caches_count): check_cleaning_parameters(caches[i], cleaning_policy, cleaning_params[i])
def test_recovery_all_options(cache_mode, cache_line_size, cleaning_policy, filesystem): """ title: Test for recovery after reset with various cache options. description: Verify that unflushed data can be safely recovered after reset. pass_criteria: - CAS recovers successfully after reboot - No data corruption """ with TestRun.step("Prepare cache and core devices."): cache_disk = TestRun.disks['cache'] core_disk = TestRun.disks['core'] cache_disk.create_partitions([Size(200, Unit.MebiByte)]) core_disk.create_partitions([Size(2000, Unit.MebiByte)] * 2) cache_device = cache_disk.partitions[0] core_device = core_disk.partitions[0] test_file = File(os.path.join(mount_point, filename)) file_operation(test_file.full_path, pattern, ReadWrite.write) file_md5 = test_file.md5sum() with TestRun.step(f"Make {filesystem} on core device."): core_device.create_filesystem(filesystem) with TestRun.step("Mount core device."): core_device.mount(mount_point) file_operation(test_file.full_path, other_pattern, ReadWrite.write) os_utils.drop_caches(DropCachesMode.ALL) with TestRun.step("Unmount core device."): core_device.unmount() with TestRun.step( f"Start cache in {cache_mode.name} with given configuration."): cache = casadm.start_cache(cache_device, cache_mode, cache_line_size, force=True) cache.set_cleaning_policy(cleaning_policy) if cleaning_policy == CleaningPolicy.acp: cache.set_params_acp( FlushParametersAcp(wake_up_time=Time(seconds=1))) with TestRun.step("Add core."): core = cache.add_core(core_device) with TestRun.step("Mount CAS device."): core.mount(mount_point) file_operation(test_file.full_path, pattern, ReadWrite.write) with TestRun.step( "Change cache mode to Write-Through without flush option."): cache.set_cache_mode(CacheMode.WT, flush=False) with TestRun.step("Reset platform."): os_utils.sync() core.unmount() TestRun.LOGGER.info( f"Number of dirty blocks in cache: {cache.get_dirty_blocks()}") power_cycle_dut() with TestRun.step("Try to start cache without load and force option."): try: casadm.start_cache(cache_device, cache_mode, cache_line_size) TestRun.fail("Cache started without load or force option.") except Exception: TestRun.LOGGER.info( "Cache did not start without load and force option.") with TestRun.step("Load cache and stop it with flush."): cache = casadm.load_cache(cache_device) cache.stop() with TestRun.step("Check md5sum of tested file on core device."): core_device.mount(mount_point) cas_md5 = test_file.md5sum() core_device.unmount() if cas_md5 == file_md5: TestRun.LOGGER.info( "Source and target file checksums are identical.") else: TestRun.fail("Source and target file checksums are different.")
def test_load_x_to_one_diff_params(cache_mode, cleaning_policy, cache_line_size, cores_amount): """ title: Test for loading CAS with 1 cache and 1 or 4 cores with different params. description: | Verify that loading cache configurations works properly in every mode with 1 cache and 1 or 4 cores. Use different parameters with load command. pass_criteria: - OpenCAS should load successfully but with saved configuration. - No errors in cache are found. """ with TestRun.step(f"Prepare 1 cache and {cores_amount} core devices"): cache_dev = TestRun.disks['cache'] cache_dev.create_partitions([Size(512, Unit.MebiByte)]) cache_dev = cache_dev.partitions[0] core_dev = TestRun.disks['core'] core_size = [] for i in range(cores_amount): core_size.append(Size(1, Unit.GibiByte)) core_dev.create_partitions(core_size) with TestRun.step(f"Start cache with {cores_amount} cores."): cache = casadm.start_cache(cache_dev, cache_mode[0], cache_line_size[0], force=True) id_cache = cache.cache_id cores = [] for i in range(cores_amount): cores.append(cache.add_core(core_dev.partitions[i])) caches_count = len(casadm_parser.get_caches()) if caches_count != 1: TestRun.fail( f"Expected caches count: 1; Actual caches count: {caches_count}." ) cores_count = len(casadm_parser.get_cores(cache.cache_id)) if cores_count != cores_amount: TestRun.fail( f"Expected cores count: {cores_amount}; Actual cores count: {cores_count}." ) with TestRun.step("Configure cleaning policy."): cache.set_cleaning_policy(cleaning_policy) if cleaning_policy == CleaningPolicy.alru: alru = FlushParametersAlru() alru.activity_threshold = Time(milliseconds=1000) alru.flush_max_buffers = 10 alru.staleness_time = Time(seconds=60) alru.wake_up_time = Time(seconds=5) cache.set_params_alru(alru) if cleaning_policy == CleaningPolicy.acp: acp = FlushParametersAcp() acp.flush_max_buffers = 100 acp.wake_up_time = Time(seconds=5) cache.set_params_acp(acp) with TestRun.step("Run FIO on exported object"): fio = (Fio().create_command().io_engine( IoEngine.libaio).io_depth(64).direct().read_write( ReadWrite.randrw).size(Size(1, Unit.GibiByte)).block_size( cache_line_size[0]).read_write(ReadWrite.randrw).num_jobs( cores_amount).cpus_allowed_policy( CpusAllowedPolicy.split)) for core in cores: fio.add_job(f"job_{core.core_id}").target(core.path) fio.run() with TestRun.step("Stop cache."): cache.stop() caches_count = len(casadm_parser.get_caches()) if caches_count != 0: TestRun.fail( f"There are still {caches_count} caches running after stopping service." ) cores_count = len(casadm_parser.get_cores(cache.cache_id)) if cores_count != 0: TestRun.fail( f"There are still {cores_count} cores running after stopping service." ) with TestRun.step("Load cache."): try: cache = casadm.start_cache(cache_dev, cache_mode[1], cache_line_size[1], id_cache + 1, load=True) except Exception: TestRun.LOGGER.error("Cannot pass other cache ID to cache load.") TestRun.LOGGER.info( "Load cache without passing cache ID to command.") cache = casadm.start_cache(cache_dev, cache_mode[1], cache_line_size[1], load=True) caches_count = len(casadm_parser.get_caches()) if caches_count != 1: TestRun.fail( f"Expected caches count: 1; Actual caches count: {caches_count}." ) cores_count = len(casadm_parser.get_cores(cache.cache_id)) if cores_count != cores_amount: TestRun.fail( f"Expected cores count: {cores_amount}; Actual cores count: {cores_count}." ) with TestRun.step("Compare cache configuration before and after load."): if cache_mode[0] != cache.get_cache_mode(): TestRun.fail("Cache modes are different. Should be the same.") if cache_line_size[0] != cache.get_cache_line_size(): TestRun.fail("Cache line sizes are different. Should be the same.") if id_cache != cache.cache_id: TestRun.fail("Cache IDs are different. Should be the same.") if cleaning_policy != cache.get_cleaning_policy(): TestRun.fail("Cleaning policies are different.") if cleaning_policy == CleaningPolicy.alru: if alru != cache.get_flush_parameters_alru(): TestRun.fail("Cleaning policy parameters are different.") if cleaning_policy == CleaningPolicy.acp: if acp != cache.get_flush_parameters_acp(): TestRun.fail("Cleaning policy parameters are different.") with TestRun.step("Run FIO again on exported object"): fio = (Fio().create_command().io_engine( IoEngine.libaio).io_depth(64).direct().read_write( ReadWrite.randrw).size(Size(1, Unit.GibiByte)).block_size( cache_line_size[1]).read_write(ReadWrite.randrw).num_jobs( cores_amount).cpus_allowed_policy( CpusAllowedPolicy.split)) for core in cores: fio.add_job(f"job_{core.core_id}").target(core.path) fio.run() with TestRun.step("Check if there are no error statistics."): if cache.get_statistics().error_stats.total_errors != 0: TestRun.fail("There are errors in the cache.") with TestRun.step("Stop caches."): casadm.stop_all_caches()