def new_cleaning_parameters_random_values(cleaning_policy):
if cleaning_policy == CleaningPolicy.alru:
alru_params_range = FlushParametersAlru().alru_params_range()
wake_up_time_random_value = Time(seconds=random.randint(
*alru_params_range.wake_up_time))
staleness_time_random_value = Time(seconds=random.randint(
*alru_params_range.staleness_time))
flush_max_buffers_random_value = random.randint(
*alru_params_range.flush_max_buffers)
activity_threshold_random_value = Time(milliseconds=random.randint(
*alru_params_range.activity_threshold))
cleaning_params = FlushParametersAlru()
cleaning_params.wake_up_time = wake_up_time_random_value
cleaning_params.staleness_time = staleness_time_random_value
cleaning_params.flush_max_buffers = flush_max_buffers_random_value
cleaning_params.activity_threshold = activity_threshold_random_value
if cleaning_policy == CleaningPolicy.acp:
acp_params_range = FlushParametersAcp().acp_params_range()
wake_up_time_random_value = Time(milliseconds=random.randint(
*acp_params_range.wake_up_time))
flush_max_buffers_random_value = random.randint(
*acp_params_range.flush_max_buffers)
cleaning_params = FlushParametersAcp()
cleaning_params.wake_up_time = wake_up_time_random_value
cleaning_params.flush_max_buffers = flush_max_buffers_random_value
return cleaning_params
def set_cleaning_policy_and_params(cache, cleaning_policy):
if cleaning_policy != CleaningPolicy.DEFAULT:
cache.set_cleaning_policy(cleaning_policy)
current_cleaning_policy = cache.get_cleaning_policy()
if current_cleaning_policy != cleaning_policy:
TestRun.LOGGER.error(f"Cleaning policy is {current_cleaning_policy}, "
f"should be {cleaning_policy}")
if cleaning_policy == CleaningPolicy.alru:
alru_params = FlushParametersAlru()
alru_params.wake_up_time = Time(seconds=10)
alru_params.staleness_time = Time(seconds=2)
alru_params.flush_max_buffers = 100
alru_params.activity_threshold = Time(milliseconds=1000)
cache.set_params_alru(alru_params)
current_alru_params = cache.get_flush_parameters_alru()
if current_alru_params != alru_params:
failed_params = ""
if current_alru_params.wake_up_time != alru_params.wake_up_time:
failed_params += (
f"Wake Up time is {current_alru_params.wake_up_time}, "
f"should be {alru_params.wake_up_time}\n")
if current_alru_params.staleness_time != alru_params.staleness_time:
failed_params += (
f"Staleness Time is {current_alru_params.staleness_time}, "
f"should be {alru_params.staleness_time}\n")
if current_alru_params.flush_max_buffers != alru_params.flush_max_buffers:
failed_params += (
f"Flush Max Buffers is {current_alru_params.flush_max_buffers}, "
f"should be {alru_params.flush_max_buffers}\n")
if current_alru_params.activity_threshold != alru_params.activity_threshold:
failed_params += (
f"Activity Threshold is {current_alru_params.activity_threshold}, "
f"should be {alru_params.activity_threshold}\n")
TestRun.LOGGER.error(
f"ALRU parameters did not switch properly:\n{failed_params}")
if cleaning_policy == CleaningPolicy.acp:
acp_params = FlushParametersAcp()
acp_params.wake_up_time = Time(milliseconds=100)
acp_params.flush_max_buffers = 64
cache.set_params_acp(acp_params)
current_acp_params = cache.get_flush_parameters_acp()
if current_acp_params != acp_params:
failed_params = ""
if current_acp_params.wake_up_time != acp_params.wake_up_time:
failed_params += (
f"Wake Up time is {current_acp_params.wake_up_time}, "
f"should be {acp_params.wake_up_time}\n")
if current_acp_params.flush_max_buffers != acp_params.flush_max_buffers:
failed_params += (
f"Flush Max Buffers is {current_acp_params.flush_max_buffers}, "
f"should be {acp_params.flush_max_buffers}\n")
TestRun.LOGGER.error(
f"ACP parameters did not switch properly:\n{failed_params}")
def prepare():
cache_dev = TestRun.disks["cache"]
core_dev = TestRun.disks["core"]
cache_dev.create_partitions([Size(100, Unit.MiB)])
core_dev.create_partitions([Size(200, Unit.MiB)])
Udev.disable()
cache = casadm.start_cache(cache_dev.partitions[0], force=True, cache_mode=CacheMode.WB)
core = cache.add_core(core_dev.partitions[0])
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
cache.set_cleaning_policy(CleaningPolicy.alru)
cache.set_params_alru(
FlushParametersAlru(
activity_threshold=Time(seconds=100),
staleness_time=Time(seconds=1),
)
)
return cache, core
def test_flush_signal_core(cache_mode):
"""
title: Test for FLUSH nad FUA signals sent to core device in modes with lazy writes.
description: |
Test if OpenCAS transmits FLUSH and FUA signals to core device in modes with lazy writes.
pass_criteria:
- FLUSH requests should be passed to core device.
- FUA requests should be passed to core device.
"""
with TestRun.step(
"Set mark in syslog to not read entries existing before the test."
):
Logs._read_syslog(Logs.last_read_line)
with TestRun.step("Prepare devices for cache and core."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(2, Unit.GibiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.scsi_debug_devices[0]
with TestRun.step(
"Start cache and add SCSI device with xfs filesystem as core."):
cache = casadm.start_cache(cache_part, cache_mode)
core_dev.create_filesystem(Filesystem.xfs)
core = cache.add_core(core_dev)
with TestRun.step("Mount exported object."):
if core.is_mounted():
core.unmount()
core.mount(mount_point)
with TestRun.step("Turn off cleaning policy."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Create temporary file on exported object."):
tmp_file = create_random_test_file(f"{mount_point}/tmp.file",
Size(1, Unit.GibiByte))
os_utils.sync()
with TestRun.step("Flush cache."):
cache.flush_cache()
os_utils.sync()
with TestRun.step(
f"Check {syslog_path} for flush request and delete temporary file."
):
Logs.check_syslog_for_signals()
tmp_file.remove(True)
with TestRun.step("Create temporary file on exported object."):
tmp_file = create_random_test_file(f"{mount_point}/tmp.file",
Size(1, Unit.GibiByte))
os_utils.sync()
with TestRun.step("Flush core."):
core.flush_core()
os_utils.sync()
with TestRun.step(
f"Check {syslog_path} for flush request and delete temporary file."
):
Logs.check_syslog_for_signals()
tmp_file.remove(True)
with TestRun.step("Turn on alru cleaning policy and set policy params."):
cache.set_cleaning_policy(CleaningPolicy.alru)
cache.set_params_alru(
FlushParametersAlru(Time(milliseconds=5000), 10000,
Time(seconds=10), Time(seconds=10)))
with TestRun.step("Create big temporary file on exported object."):
tmp_file = create_random_test_file(f"{mount_point}/tmp.file",
Size(5, Unit.GibiByte))
os_utils.sync()
with TestRun.step(
"Wait for automatic flush from alru cleaning policy and check log."
):
wait_time = (int(
cache.get_flush_parameters_alru().staleness_time.total_seconds()) +
int(cache.get_flush_parameters_alru().activity_threshold.
total_seconds()) +
int(cache.get_flush_parameters_alru().wake_up_time.
total_seconds()) + 5)
sleep(wait_time)
with TestRun.step(
f"Check {syslog_path} for flush request and delete temporary file."
):
Logs.check_syslog_for_signals()
tmp_file.remove(True)
with TestRun.step("Create temporary file on exported object."):
create_random_test_file(f"{mount_point}/tmp.file",
Size(1, Unit.GibiByte))
os_utils.sync()
with TestRun.step("Unmount exported object and remove it from cache."):
core.unmount()
core.remove_core()
os_utils.sync()
with TestRun.step(f"Check {syslog_path} for flush request."):
Logs.check_syslog_for_signals()
with TestRun.step("Stop cache."):
cache.stop()
def test_set_get_cleaning_params(cache_mode, cleaning_policy):
"""
title: Test for setting and reading cleaning parameters.
description: |
Verify that it is possible to set and read all available cleaning
parameters for all cleaning policies using casadm --set-param and
--get-param options.
pass_criteria:
- All cleaning parameters are set to given values.
- All cleaning parameters displays proper values.
"""
with TestRun.step("Partition cache and core devices"):
cache_dev, core_dev = storage_prepare()
with TestRun.step(
f"Start {caches_count} caches in {cache_mode} cache mode "
f"and add {cores_per_cache} cores per cache"):
caches, cores = cache_prepare(cache_mode, cache_dev, core_dev)
with TestRun.step(f"Set cleaning policy to {cleaning_policy}"):
if cleaning_policy != CleaningPolicy.DEFAULT:
for i in range(caches_count):
caches[i].set_cleaning_policy(cleaning_policy)
for i in range(caches_count):
current_cleaning_policy = caches[i].get_cleaning_policy()
if current_cleaning_policy != cleaning_policy:
TestRun.fail(
f"Cleaning policy for cache nr {caches[i].cache_id} "
f"is {current_cleaning_policy}, should be {cleaning_policy}"
)
with TestRun.step(
f"Check {cleaning_policy} cleaning policy default parameters"):
if cleaning_policy == CleaningPolicy.alru:
default_cleaning_params = FlushParametersAlru.default_alru_params()
if cleaning_policy == CleaningPolicy.acp:
default_cleaning_params = FlushParametersAcp.default_acp_params()
for i in range(caches_count):
check_cleaning_parameters(caches[i], cleaning_policy,
default_cleaning_params)
with TestRun.step(f"Set new random values for {cleaning_policy} "
f"cleaning policy parameters for one cache instance"):
for check in range(number_of_checks):
random_cleaning_params = new_cleaning_parameters_random_values(
cleaning_policy)
if cleaning_policy == CleaningPolicy.alru:
caches[0].set_params_alru(random_cleaning_params)
if cleaning_policy == CleaningPolicy.acp:
caches[0].set_params_acp(random_cleaning_params)
# Check changed parameters for first cache instance:
check_cleaning_parameters(caches[0], cleaning_policy,
random_cleaning_params)
# Check default parameters for other cache instances:
for i in range(1, caches_count):
check_cleaning_parameters(caches[i], cleaning_policy,
default_cleaning_params)
with TestRun.step(f"Set new random values for {cleaning_policy} "
f"cleaning policy parameters for all cache instances"):
for check in range(number_of_checks):
cleaning_params = []
for i in range(caches_count):
random_cleaning_params = new_cleaning_parameters_random_values(
cleaning_policy)
cleaning_params.append(random_cleaning_params)
if cleaning_policy == CleaningPolicy.alru:
caches[i].set_params_alru(random_cleaning_params)
if cleaning_policy == CleaningPolicy.acp:
caches[i].set_params_acp(random_cleaning_params)
for i in range(caches_count):
check_cleaning_parameters(caches[i], cleaning_policy,
cleaning_params[i])
def test_load_x_to_one_diff_params(cache_mode, cleaning_policy,
cache_line_size, cores_amount):
"""
title: Test for loading CAS with 1 cache and 1 or 4 cores with different params.
description: |
Verify that loading cache configurations works properly in every mode
with 1 cache and 1 or 4 cores. Use different parameters with load command.
pass_criteria:
- OpenCAS should load successfully but with saved configuration.
- No errors in cache are found.
"""
with TestRun.step(f"Prepare 1 cache and {cores_amount} core devices"):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_size = []
for i in range(cores_amount):
core_size.append(Size(1, Unit.GibiByte))
core_dev.create_partitions(core_size)
with TestRun.step(f"Start cache with {cores_amount} cores."):
cache = casadm.start_cache(cache_dev,
cache_mode[0],
cache_line_size[0],
force=True)
id_cache = cache.cache_id
cores = []
for i in range(cores_amount):
cores.append(cache.add_core(core_dev.partitions[i]))
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != cores_amount:
TestRun.fail(
f"Expected cores count: {cores_amount}; Actual cores count: {cores_count}."
)
with TestRun.step("Configure cleaning policy."):
cache.set_cleaning_policy(cleaning_policy)
if cleaning_policy == CleaningPolicy.alru:
alru = FlushParametersAlru()
alru.activity_threshold = Time(milliseconds=1000)
alru.flush_max_buffers = 10
alru.staleness_time = Time(seconds=60)
alru.wake_up_time = Time(seconds=5)
cache.set_params_alru(alru)
if cleaning_policy == CleaningPolicy.acp:
acp = FlushParametersAcp()
acp.flush_max_buffers = 100
acp.wake_up_time = Time(seconds=5)
cache.set_params_acp(acp)
with TestRun.step("Run FIO on exported object"):
fio = (Fio().create_command().io_engine(
IoEngine.libaio).io_depth(64).direct().read_write(
ReadWrite.randrw).size(Size(1, Unit.GibiByte)).block_size(
cache_line_size[0]).read_write(ReadWrite.randrw).num_jobs(
cores_amount).cpus_allowed_policy(
CpusAllowedPolicy.split))
for core in cores:
fio.add_job(f"job_{core.core_id}").target(core.path)
fio.run()
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"There are still {caches_count} caches running after stopping service."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(
f"There are still {cores_count} cores running after stopping service."
)
with TestRun.step("Load cache."):
try:
cache = casadm.start_cache(cache_dev,
cache_mode[1],
cache_line_size[1],
id_cache + 1,
load=True)
except Exception:
TestRun.LOGGER.error("Cannot pass other cache ID to cache load.")
TestRun.LOGGER.info(
"Load cache without passing cache ID to command.")
cache = casadm.start_cache(cache_dev,
cache_mode[1],
cache_line_size[1],
load=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != cores_amount:
TestRun.fail(
f"Expected cores count: {cores_amount}; Actual cores count: {cores_count}."
)
with TestRun.step("Compare cache configuration before and after load."):
if cache_mode[0] != cache.get_cache_mode():
TestRun.fail("Cache modes are different. Should be the same.")
if cache_line_size[0] != cache.get_cache_line_size():
TestRun.fail("Cache line sizes are different. Should be the same.")
if id_cache != cache.cache_id:
TestRun.fail("Cache IDs are different. Should be the same.")
if cleaning_policy != cache.get_cleaning_policy():
TestRun.fail("Cleaning policies are different.")
if cleaning_policy == CleaningPolicy.alru:
if alru != cache.get_flush_parameters_alru():
TestRun.fail("Cleaning policy parameters are different.")
if cleaning_policy == CleaningPolicy.acp:
if acp != cache.get_flush_parameters_acp():
TestRun.fail("Cleaning policy parameters are different.")
with TestRun.step("Run FIO again on exported object"):
fio = (Fio().create_command().io_engine(
IoEngine.libaio).io_depth(64).direct().read_write(
ReadWrite.randrw).size(Size(1, Unit.GibiByte)).block_size(
cache_line_size[1]).read_write(ReadWrite.randrw).num_jobs(
cores_amount).cpus_allowed_policy(
CpusAllowedPolicy.split))
for core in cores:
fio.add_job(f"job_{core.core_id}").target(core.path)
fio.run()
with TestRun.step("Check if there are no error statistics."):
if cache.get_statistics().error_stats.total_errors != 0:
TestRun.fail("There are errors in the cache.")
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
def test_flush_inactive_devices():
"""
title: Negative test for flushing inactive CAS devices.
description: Validate that CAS prevents flushing dirty data from inactive CAS devices.
pass_criteria:
- No kernel error
- Exported object appears after plugging core device
- Flushing inactive CAS devices is possible neither by cleaning thread,
nor by calling cleaning methods
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core1", 1), ("core2", 1)])
cache_dev = devices["cache"].partitions[0]
first_core_dev = devices["core1"].partitions[0]
second_core_dev = devices["core2"].partitions[0]
plug_device = devices["core1"]
with TestRun.step("Start cache in WB mode and set alru cleaning policy."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cache.set_cleaning_policy(CleaningPolicy.alru)
cache.set_params_alru(
FlushParametersAlru(staleness_time=Time(seconds=10),
wake_up_time=Time(seconds=1),
activity_threshold=Time(milliseconds=500)))
with TestRun.step("Add two cores."):
first_core = cache.add_core(first_core_dev)
second_core = cache.add_core(second_core_dev)
with TestRun.step(
"Create init config file using running CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run random writes to CAS device."):
run_fio([first_core.path, second_core.path])
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug one core disk."):
plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Wait longer than required for alru cleaning thread to start and verify "
"that dirty data is flushed only from active device."):
dirty_lines_before = {
first_core: first_core.get_dirty_blocks(),
second_core: second_core.get_dirty_blocks()
}
time.sleep(30)
check_amount_of_dirty_data(dirty_lines_before)
with TestRun.step("Try to call 'flush cache' command."):
dirty_lines_before = {
first_core: first_core.get_dirty_blocks(),
second_core: second_core.get_dirty_blocks()
}
try:
cache.flush_cache()
TestRun.fail(
"Flush cache operation should be blocked due to inactive cache devices, "
"but it executed successfully.")
except Exception as e:
TestRun.LOGGER.info(
f"Flush cache operation is blocked as expected.\n{str(e)}")
check_amount_of_dirty_data(dirty_lines_before)
with TestRun.step("Try to call 'flush core' command for inactive core."):
dirty_lines_before = {
first_core: first_core.get_dirty_blocks(),
second_core: second_core.get_dirty_blocks()
}
try:
first_core.flush_core()
TestRun.fail(
"Flush core operation should be blocked for inactive CAS devices, "
"but it executed successfully.")
except Exception as e:
TestRun.LOGGER.info(
f"Flush core operation is blocked as expected.\n{str(e)}")
check_amount_of_dirty_data(dirty_lines_before)
with TestRun.step(
"Plug core disk and verify that this change is reflected on the cache list."
):
plug_device.plug()
time.sleep(1)
first_core.wait_for_status_change(CoreStatus.active)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache did not change status to 'running' after plugging core device. "
f"Actual state: {cache_status}.")
with TestRun.step("Stop cache."):
cache.stop()
def test_alru_no_idle():
"""
title: Test ALRU with activity threshold set to 0
description: |
Verify that ALRU is able to perform cleaning if cache is under constant load and
activity threshold is set to 0. Constant load is performed by using fio instance running
in background.
pass_criteria:
- Dirty cache lines are cleaned successfuly.
"""
with TestRun.step("Prepare configuration"):
cache, core = prepare()
with TestRun.step("Prepare dirty data to be cleaned"):
bg_size = Size(2, Unit.MiB)
(
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.offset(bg_size)
.size(Size(10, Unit.MiB))
.block_size(Size(4, Unit.KiB))
.target(core)
.direct()
.read_write(ReadWrite.randwrite)
.run()
)
with TestRun.step("Run background fio"):
(
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.size(bg_size)
.block_size(Size(4, Unit.KiB))
.target(core)
.direct()
.time_based(True)
.run_time(timedelta(hours=1))
.read_write(ReadWrite.randwrite)
.run_in_background()
)
with TestRun.step("Verify that cache is dirty"):
# Wait for bg fio to dirty whole workset
time.sleep(5)
dirty_before = cache.get_statistics().usage_stats.dirty
if dirty_before == Size(0):
TestRun.fail("Cache should be dirty")
with TestRun.step("Check that cleaning doesn't occur under constant load"):
time.sleep(5)
dirty_now = cache.get_statistics().usage_stats.dirty
if dirty_before > dirty_now:
TestRun.fail(
f"Cleaning has run, while it shouldn't"
" (dirty down from {dirty_before} to {dirty_now}"
)
with TestRun.step("Set 0 idle time and wake up time for ALRU"):
cache.set_params_alru(FlushParametersAlru(activity_threshold=Time(0), wake_up_time=Time(0)))
with TestRun.step("Check that cleaning is progressing"):
time.sleep(5)
if dirty_before <= cache.get_statistics().usage_stats.dirty:
TestRun.fail("Cleaning didn't run")
kill_all_io()
