def test_stress_add_remove_core(cache_mode):
"""
title: Stress test for adding and removing core.
description: Validate the ability of CAS to add and remove core in the loop.
pass_criteria:
- No system crash while adding and removing core.
- Core is added and removed successfully.
- No data corruption.
"""
with TestRun.step(
"Prepare cache and core. Create test file and count its checksum."
):
cache, core, file, file_md5sum_before = prepare_with_file_creation(
cache_mode)
for _ in TestRun.iteration(
range(0, iterations_per_config),
f"Add and remove core {iterations_per_config} times."):
with TestRun.step("Remove core."):
core.remove_core()
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(
f"Expected cores count: 0; Actual cores count: {cores_count}."
)
with TestRun.step("Add core."):
core = cache.add_core(core.core_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}."
)
with TestRun.step("Check md5 sum of test file."):
core.mount(mount_point)
file_md5sum_after = file.md5sum()
if file_md5sum_after != file_md5sum_before:
TestRun.LOGGER.error("Md5 sum of test file is different.")
core.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_stress_service(cache_mode):
"""
title: Stress test for starting and stopping CAS service.
description: |
Validate the ability of CAS to restart CAS service
and load CAS device in the loop.
pass_criteria:
- No system crash while restarting CAS service or loading cache.
- CAS service restarts with no errors.
- CAS device loads successfully.
"""
with TestRun.step("Prepare cache and core."):
cache_dev, core_dev = prepare()
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
casadm.add_core(cache, core_dev)
for _ in TestRun.iteration(
range(0, iterations_per_config),
f"Stop and start CAS service {iterations_per_config} times."):
with TestRun.step(
"Create CAS init config based on current running CAS configuration."
):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop CAS service."):
casctl.stop()
with TestRun.step("Check if service stopped successfully."):
if len(casadm_parser.get_caches()) != 0:
TestRun.fail(
"There are still running caches after stopping service.")
if len(casadm_parser.get_cores(cache.cache_id)) != 0:
TestRun.fail(
"There are still running cores after stopping service.")
with TestRun.step("Start CAS service."):
casctl.start()
time.sleep(1) # Time for CAS devices to start
with TestRun.step("Check if CAS configuration loaded successfully."):
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}."
)
with TestRun.step("Stop caches and create default init config file."):
casadm.stop_all_caches()
InitConfig.create_default_init_config()
def test_stress_reload_module(cache_mode):
"""
title: Stress test for reloading CAS modules.
description: Validate the ability of CAS to reload modules in the loop.
pass_criteria:
- No system crash while reloading CAS modules.
- CAS modules reloads with no errors.
- No data corruption.
"""
with TestRun.step("Prepare cache and core. Create test file and count its checksum."):
cache, core, file, file_md5sum_before = prepare_with_file_creation(cache_mode)
with TestRun.step("Save current cache configuration."):
cache_config = cache.get_cache_config()
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Reload CAS modules and check loaded "
f"cache configuration {iterations_per_config} times."):
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(f"Expected cores count: 0; Actual cores count: {cores_count}.")
with TestRun.step("Reload CAS modules."):
cas_module.reload_all_cas_modules()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Validate cache configuration."):
if cache.get_cache_config() != cache_config:
TestRun.fail("Cache configuration is different than before reloading modules.")
with TestRun.step("Check md5 sum of test file."):
core.mount(mount_point)
file_md5sum_after = file.md5sum()
if file_md5sum_after != file_md5sum_before:
TestRun.LOGGER.error("Md5 sum of test file is different.")
core.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_stress_load(cache_mode):
"""
title: Stress test for stopping and loading CAS device.
description: |
Validate the ability of the CAS to load and stop cache in the loop
using different cache modes.
pass_criteria:
- No system crash while stop and load cache in the loop.
- CAS device loads successfully.
"""
with TestRun.step("Prepare cache and core."):
cache_dev, core_dev = prepare()
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
casadm.add_core(cache, core_dev)
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Stop cache and load it {iterations_per_config} times."):
with TestRun.step("Stop cache."):
casadm.stop_cache(cache.cache_id)
if len(casadm_parser.get_caches()) != 0:
TestRun.fail("Cache did not stop successfully.")
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Cache did not load successfully - wrong number of caches: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.LOGGER.error(f"Cache loaded with wrong cores count: {cores_count}.")
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_stress_reload_cache(cache_mode):
"""
title: Stress test for reloading cache with simple data integrity check.
description: |
Validate the ability of CAS to reload cache in the loop
with no data corruption.
pass_criteria:
- No system crash while reloading cache.
- CAS device loads successfully.
- No data corruption.
"""
with TestRun.step("Prepare cache and core. Create test file and count it's checksum."):
cache, core, md5_before_load, size_before_load, permissions_before_load = \
prepare_with_file_creation(cache_mode)
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Stop and load cache {iterations_per_config} times."):
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Check md5 of test file."):
check_files(core, size_before_load, permissions_before_load, md5_before_load)
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_stress_start(cache_mode):
"""
title: Stress test for starting and stopping cache.
description: Validate the ability of CAS to start and stop cache in the loop.
pass_criteria:
- No system crash while starting and stopping cache in the loop.
- Cache starts and stops successfully.
"""
with TestRun.step("Prepare cache and core."):
cache_dev, core_dev = prepare()
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Start and stop CAS {iterations_per_config} times."):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
with TestRun.step("Add core."):
cache.add_core(core_dev)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_dirty_load():
"""
title: Loading cache after dirty shutdown.
description: Test for loading cache containing dirty data after DUT hard restart.
pass_criteria:
- DUT should reboot successfully.
- Cache should load successfully.
"""
with TestRun.step("Prepare devices."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks['core']
core_disk.create_partitions([Size(2, Unit.GibiByte)] * 2)
core_devices = core_disk.partitions
with TestRun.step("Start cache in Write-Back mode and add cores."):
cache = casadm.start_cache(cache_dev, cache_mode=CacheMode.WB)
cores = []
for dev in core_devices:
cores.append(cache.add_core(dev))
with TestRun.step("Set cleaning policy to nop."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Populate cache with dirty data."):
fio = Fio().create_command()\
.size(Size(1, Unit.GibiByte))\
.read_write(ReadWrite.randwrite)\
.io_engine(IoEngine.libaio)\
.block_size(Size(1, Unit.Blocks4096))
for i, core in enumerate(cores):
fio.add_job(f"core{i}").target(core.path)
fio.run()
if cache.get_dirty_blocks() <= Size.zero():
TestRun.fail("Cache does not contain dirty data.")
with TestRun.step("Remove one core without flushing dirty data."):
casadm.remove_core_with_script_command(cache.cache_id, core.core_id,
True)
with TestRun.step("Reset platform."):
power_control = TestRun.plugin_manager.get_plugin('power_control')
power_control.power_cycle()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
caches_num = len(casadm_parser.get_caches())
if caches_num != 1:
TestRun.LOGGER.error(
f"Wrong number of caches. Expected 1, actual {caches_num}.")
cores_num = len(casadm_parser.get_cores(cache.cache_id))
if cores_num != 1:
TestRun.LOGGER.error(
f"Wrong number of cores. Expected 1, actual {cores_num}.")
def test_one_core_release(cache_mode):
"""
title: Test if OpenCAS dynamically allocates space according to core devices needs.
description: |
When one or more core devices are unused in a single cache instance all blocks
previously occupied should be available to other core devices.
Test is without pass through mode.
pass_criteria:
- No system crash.
- The remaining core is able to use cache.
- OpenCAS frees blocks occupied by unused core and allocates it to the remaining core.
"""
with TestRun.step("Prepare two cache and one core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte)] * 2)
core_part1 = core_dev.partitions[0]
core_part2 = core_dev.partitions[1]
Udev.disable()
with TestRun.step("Start cache"):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
with TestRun.step("Add both core devices to cache."):
core1 = cache.add_core(core_part1)
core2 = cache.add_core(core_part2)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 2:
TestRun.fail(f"Expected cores count: 2; Actual cores count: {cores_count}.")
with TestRun.step("Change sequential cutoff policy to 'never'."):
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Fill cache with pages from the first core."):
dd_builder(cache_mode, core1, cache.size).run()
core1_occupied_blocks_before = core1.get_occupancy()
with TestRun.step("Check if the remaining core is able to use cache."):
dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run()
core1_occupied_blocks_after = core1.get_occupancy()
with TestRun.step("Check if occupancy from the first core is removed from cache."):
# The first core's occupancy should be lower than cache's occupancy
# by the value of the remaining core's occupancy because cache
# should reallocate blocks from unused core to used core.
if core1_occupied_blocks_after >= core1_occupied_blocks_before \
or cache.get_occupancy() <= core1_occupied_blocks_after \
or not float(core2.get_occupancy().get_value()) > 0:
TestRun.LOGGER.error("Blocks previously occupied by the first core aren't released.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_load_cache_with_mounted_core(cache_mode):
"""
title: Fault injection test for adding mounted core on cache load.
description: |
Negative test of the ability of CAS to add to cache while its loading
core device which is mounted.
pass_criteria:
- No system crash while loading cache.
- Adding mounted core while loading cache fails.
"""
with TestRun.step("Prepare cache and core devices. Start CAS."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_part = core_dev.partitions[0]
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Add core device with xfs filesystem to cache and mount it."):
core_part.create_filesystem(Filesystem.xfs)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(f"Create test file in mount point of exported object and check its md5 sum."):
test_file = fs_utils.create_random_test_file(test_file_path)
test_file_md5_before = test_file.md5sum()
with TestRun.step("Unmount core device."):
core.unmount()
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
with TestRun.step("Mount core device."):
core_part.mount(mount_point)
with TestRun.step("Try to load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1 Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(f"Expected cores count: 0; Actual cores count: {cores_count}.")
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5_before != test_file.md5sum():
TestRun.LOGGER.error("Md5 sum of test file is different.")
core_part.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def create_init_config_from_running_configuration(cls,
load: bool = None,
extra_flags=""):
init_conf = cls()
for cache in casadm_parser.get_caches():
init_conf.add_cache(cache.cache_id, cache.cache_device,
cache.get_cache_mode(), load, extra_flags)
for core in casadm_parser.get_cores(cache.cache_id):
init_conf.add_core(cache.cache_id, core.core_id,
core.core_device)
init_conf.save_config_file()
return init_conf
def test_stress_add_remove_core(cache_mode):
"""
title: Stress test for adding and removing core.
description: Validate the ability of CAS to add and remove core in the loop.
pass_criteria:
- No system crash while adding and removing core.
- Core is added and removed successfully.
- No data corruption.
"""
with TestRun.step("Prepare cache and core. Create test file and count it's checksum."):
cache, core, md5_before_load, size_before_load, permissions_before_load = \
prepare_with_file_creation(cache_mode)
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Add and remove core {iterations_per_config} times."):
with TestRun.step("Remove core."):
core.remove_core()
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(f"Expected cores count: 0; Actual cores count: {cores_count}.")
with TestRun.step("Add core."):
core = cache.add_core(core.core_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Check md5 of test file."):
check_files(core, size_before_load, permissions_before_load, md5_before_load)
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def validate_cache(cache_mode):
caches = casadm_parser.get_caches()
caches_count = len(caches)
if caches_count != 1:
TestRun.LOGGER.error(
f"Cache did not start successfully - wrong number of caches: {caches_count}."
)
cores = casadm_parser.get_cores(caches[0].cache_id)
cores_count = len(cores)
if cores_count != 1:
TestRun.LOGGER.error(f"Cache started with wrong number of cores: {cores_count}.")
current_mode = caches[0].get_cache_mode()
if current_mode != cache_mode:
TestRun.LOGGER.error(
f"Cache started in wrong mode!\n"
f"Should start in {cache_mode}, but started in {current_mode} mode."
)
def check_configuration(cleaning_policy, cache_mode, cache_line_size):
caches = casadm_parser.get_caches()
if len(caches) != 1:
TestRun.fail(f"There is wrong amount of caches running. "
f"(Expected: 1, actual: {len(caches)}).")
actual_cores_number = len(casadm_parser.get_cores(caches[0].cache_id))
if actual_cores_number != cores_number:
TestRun.fail(f"There is wrong amount of CAS devices running. (Expected: {cores_number}, "
f"actual: {actual_cores_number}).")
actual_line_size = caches[0].get_cache_line_size()
actual_cache_mode = caches[0].get_cache_mode()
actual_cleaning_policy = caches[0].get_cleaning_policy()
if actual_cleaning_policy != cleaning_policy:
TestRun.fail(f"Cleaning policy: expected = {cleaning_policy.value}, "
f"actual = {actual_cleaning_policy.value}.")
if actual_cache_mode != cache_mode:
TestRun.fail(f"Cache mode: expected = {cache_mode.value}, actual = {actual_cache_mode}.")
if actual_line_size != cache_line_size:
TestRun.fail(f"Cache line size: expected = {cache_line_size.name}, "
f"actual: {actual_line_size.name}")
def create_init_config_from_running_configuration(load: bool = None,
extra_flags=""):
cache_lines = []
core_lines = []
for cache in casadm_parser.get_caches():
cache_lines.append(
CacheConfigLine(cache.cache_id, cache.cache_device,
cache.get_cache_mode(), load, extra_flags))
for core in casadm_parser.get_cores(cache.cache_id):
core_lines.append(
CoreConfigLine(cache.cache_id, core.core_id, core.core_device))
config_lines = []
create_default_init_config()
if len(cache_lines) > 0:
config_lines.append(CacheConfigLine.header)
for c in cache_lines:
config_lines.append(str(c))
if len(core_lines) > 0:
config_lines.append(CoreConfigLine.header)
for c in core_lines:
config_lines.append(str(c))
fs_utils.write_file(cas_init_config_path, '\n'.join(config_lines), False)
def test_load_x_to_one_diff_params(cache_mode, cleaning_policy,
cache_line_size, cores_amount):
"""
title: Test for loading CAS with 1 cache and 1 or 4 cores with different params.
description: |
Verify that loading cache configurations works properly in every mode
with 1 cache and 1 or 4 cores. Use different parameters with load command.
pass_criteria:
- OpenCAS should load successfully but with saved configuration.
- No errors in cache are found.
"""
with TestRun.step(f"Prepare 1 cache and {cores_amount} core devices"):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_size = []
for i in range(cores_amount):
core_size.append(Size(1, Unit.GibiByte))
core_dev.create_partitions(core_size)
with TestRun.step(f"Start cache with {cores_amount} cores."):
cache = casadm.start_cache(cache_dev,
cache_mode[0],
cache_line_size[0],
force=True)
id_cache = cache.cache_id
cores = []
for i in range(cores_amount):
cores.append(cache.add_core(core_dev.partitions[i]))
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != cores_amount:
TestRun.fail(
f"Expected cores count: {cores_amount}; Actual cores count: {cores_count}."
)
with TestRun.step("Configure cleaning policy."):
cache.set_cleaning_policy(cleaning_policy)
if cleaning_policy == CleaningPolicy.alru:
alru = FlushParametersAlru()
alru.activity_threshold = Time(milliseconds=1000)
alru.flush_max_buffers = 10
alru.staleness_time = Time(seconds=60)
alru.wake_up_time = Time(seconds=5)
cache.set_params_alru(alru)
if cleaning_policy == CleaningPolicy.acp:
acp = FlushParametersAcp()
acp.flush_max_buffers = 100
acp.wake_up_time = Time(seconds=5)
cache.set_params_acp(acp)
with TestRun.step("Run FIO on exported object"):
fio = (Fio().create_command().io_engine(
IoEngine.libaio).io_depth(64).direct().read_write(
ReadWrite.randrw).size(Size(1, Unit.GibiByte)).block_size(
cache_line_size[0]).read_write(ReadWrite.randrw).num_jobs(
cores_amount).cpus_allowed_policy(
CpusAllowedPolicy.split))
for core in cores:
fio.add_job(f"job_{core.core_id}").target(core.path)
fio.run()
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"There are still {caches_count} caches running after stopping service."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(
f"There are still {cores_count} cores running after stopping service."
)
with TestRun.step("Load cache."):
try:
cache = casadm.start_cache(cache_dev,
cache_mode[1],
cache_line_size[1],
id_cache + 1,
load=True)
except Exception:
TestRun.LOGGER.error("Cannot pass other cache ID to cache load.")
TestRun.LOGGER.info(
"Load cache without passing cache ID to command.")
cache = casadm.start_cache(cache_dev,
cache_mode[1],
cache_line_size[1],
load=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != cores_amount:
TestRun.fail(
f"Expected cores count: {cores_amount}; Actual cores count: {cores_count}."
)
with TestRun.step("Compare cache configuration before and after load."):
if cache_mode[0] != cache.get_cache_mode():
TestRun.fail("Cache modes are different. Should be the same.")
if cache_line_size[0] != cache.get_cache_line_size():
TestRun.fail("Cache line sizes are different. Should be the same.")
if id_cache != cache.cache_id:
TestRun.fail("Cache IDs are different. Should be the same.")
if cleaning_policy != cache.get_cleaning_policy():
TestRun.fail("Cleaning policies are different.")
if cleaning_policy == CleaningPolicy.alru:
if alru != cache.get_flush_parameters_alru():
TestRun.fail("Cleaning policy parameters are different.")
if cleaning_policy == CleaningPolicy.acp:
if acp != cache.get_flush_parameters_acp():
TestRun.fail("Cleaning policy parameters are different.")
with TestRun.step("Run FIO again on exported object"):
fio = (Fio().create_command().io_engine(
IoEngine.libaio).io_depth(64).direct().read_write(
ReadWrite.randrw).size(Size(1, Unit.GibiByte)).block_size(
cache_line_size[1]).read_write(ReadWrite.randrw).num_jobs(
cores_amount).cpus_allowed_policy(
CpusAllowedPolicy.split))
for core in cores:
fio.add_job(f"job_{core.core_id}").target(core.path)
fio.run()
with TestRun.step("Check if there are no error statistics."):
if cache.get_statistics().error_stats.total_errors != 0:
TestRun.fail("There are errors in the cache.")
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
def test_cas_startup(cache_mode, filesystem):
"""
title: Test for starting CAS on system startup.
pass_criteria:
- System does not crash.
- CAS modules are loaded before partitions are mounted.
- Cache is loaded before partitions are mounted.
- Exported object is mounted after startup is complete.
"""
with TestRun.step(
"Prepare partitions for cache (200MiB) and for core (400MiB)"):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(200, Unit.MebiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(400, Unit.MebiByte)])
core_part = core_dev.partitions[0]
with TestRun.step("Start cache and add core"):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
core = cache.add_core(core_part)
with TestRun.step("Create and mount filesystem"):
core.create_filesystem(filesystem)
core.mount(mountpoint)
with TestRun.step("Create test file and calculate md5 checksum"):
(Dd().input("/dev/urandom").output(filepath).count(16).block_size(
Size(1, Unit.MebiByte)).run())
test_file = File(filepath)
md5_before = test_file.md5sum()
with TestRun.step("Add mountpoint fstab and create intelcas.conf"):
fstab.add_mountpoint(device=core,
mount_point=mountpoint,
fs_type=filesystem)
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Reboot"):
TestRun.executor.reboot()
with TestRun.step("Check if cache is started"):
caches = list(get_caches())
if len(caches) != 1:
TestRun.fail(f"Expected one cache, got {len(caches)}!")
if caches[0].cache_id != cache.cache_id:
TestRun.fail("Invalid cache id!")
with TestRun.step("Check if core is added"):
cores = list(get_cores(cache.cache_id))
if len(cores) != 1:
TestRun.fail(f"Expected one core, got {len(cores)}!")
if cores[0].core_id != core.core_id:
TestRun.fail("Invalid core id!")
with TestRun.step("Check if filesystem is mounted"):
if not core.is_mounted():
TestRun.fail("Core is not mounted!")
with TestRun.step("Check if md5 checksum matches"):
md5_after = test_file.md5sum()
if md5_before != md5_after:
TestRun.fail("md5 checksum mismatch!")
with TestRun.step("Test cleanup"):
fstab.remove_mountpoint(device=core)
core.unmount()
InitConfig.create_default_init_config()
casadm.stop_all_caches()
def test_one_core_fail(cache_mode):
"""
title: Test if OpenCAS correctly handles failure of one of multiple core devices.
description: |
When one core device fails in a single cache instance all blocks previously occupied
should be available to other core devices.
Test is without pass through mode.
pass_criteria:
- No system crash.
- Second core is able to use OpenCAS.
"""
with TestRun.step("Prepare one cache and two core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)] * 2)
cache_part = cache_dev.partitions[0]
core_dev1 = TestRun.disks['core1'] # This device would be unplugged.
core_dev1.create_partitions([Size(2, Unit.GibiByte)])
core_part1 = core_dev1.partitions[0]
core_dev2 = TestRun.disks['core2']
core_dev2.create_partitions([Size(2, Unit.GibiByte)])
core_part2 = core_dev2.partitions[0]
Udev.disable()
with TestRun.step("Start cache"):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
with TestRun.step("Add both core devices to cache."):
core1 = cache.add_core(core_part1)
core2 = cache.add_core(core_part2)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 2:
TestRun.fail(
f"Expected cores count: 2; Actual cores count: {cores_count}.")
with TestRun.step("Change sequential cutoff policy."):
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Fill cache with pages from the first core."):
dd_builder(cache_mode, core1, cache.size).run()
cache_occupied_blocks_before = cache.get_occupancy()
with TestRun.step("Unplug the first core device."):
core_dev1.unplug()
with TestRun.step("Check if core device is really out of cache."):
output = str(casadm.list_caches().stdout.splitlines())
if core_part1.path in output:
TestRun.fail("The first core device should be unplugged!")
with TestRun.step("Check if the remaining core is able to use cache."):
dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run()
if not float(core2.get_occupancy().get_value()) > 0:
TestRun.LOGGER.error(
"The remaining core is not able to use cache.")
with TestRun.step(
"Check if occupancy from the first core is removed from cache."):
# Cache occupancy cannot be lower than before the first core fails and after that
# should be equal to the sum of occupancy of the first and the remaining core
cache_occupied_blocks_after = cache.get_occupancy()
if cache_occupied_blocks_before > cache_occupied_blocks_after \
or cache_occupied_blocks_after != core2.get_occupancy() + core1.get_occupancy():
TestRun.fail("Blocks previously occupied by the first core "
"aren't released after this core failure.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
with TestRun.step("Plug back the first core."):
core_dev1.plug()
def test_one_core_remove(cache_mode):
"""
title: Test if OpenCAS correctly handles removal of one of multiple core devices.
description: |
When one core device is removed from a cache instance all blocks previously occupied
by the data from that core device should be removed. That means that the number of free
cache blocks should increase by the number of removed blocks.
Test is without pass through mode.
pass_criteria:
- No system crash.
- The remaining core is able to use cache.
- Removing core frees cache blocks occupied by this core.
"""
with TestRun.step("Prepare one device for cache and two for core."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte)] * 2)
core_part1 = core_dev.partitions[0]
core_part2 = core_dev.partitions[1]
Udev.disable()
with TestRun.step("Start cache"):
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
with TestRun.step("Add both core devices to cache."):
core1 = cache.add_core(core_part1)
core2 = cache.add_core(core_part2)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 2:
TestRun.fail(
f"Expected cores count: 2; Actual cores count: {cores_count}.")
with TestRun.step("Fill cache with pages from the first core."):
dd_builder(cache_mode, core1, cache.size).run()
core1_occupied_blocks = core1.get_occupancy()
occupied_blocks_before = cache.get_occupancy()
with TestRun.step("Remove the first core."):
cache.remove_core(core1.core_id, True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step(
"Check if occupancy from the first core is removed from cache."):
# Blocks occupied by the first core should be completely released.
if cache.get_occupancy(
) != occupied_blocks_before - core1_occupied_blocks:
TestRun.LOGGER.error(
"Blocks previously occupied by the first core "
"aren't released by removing this core.")
with TestRun.step("Check if the remaining core is able to use cache."):
dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run()
if not float(core2.get_occupancy().get_value()) > 0:
TestRun.fail("The remaining core is not able to use cache.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_stop_no_flush_load_cache_no_fs(cache_mode):
"""
title: Test to check that 'stop --no-data-flush' command works correctly.
description: |
Negative test of the ability of CAS to load unflushed cache on core device
without filesystem. Test uses lazy flush cache modes.
pass_criteria:
- No system crash while load cache.
- Starting cache without loading metadata fails.
- Starting cache with loading metadata finishes with success.
"""
with TestRun.step("Prepare cache and core devices."):
cache_part, core_part = prepare()
with TestRun.step("Start cache with --force option."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Change cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Add core device without filesystem."):
core_part.wipe_filesystem()
core = cache.add_core(core_part)
with TestRun.step("Fill exported object with data."):
dd = (Dd().input("/dev/zero").output(core.path).block_size(
Size(1, Unit.Blocks4096)).oflag("direct"))
dd.run()
with TestRun.step("Count dirty blocks on exported object."):
dirty_blocks_before = core.get_dirty_blocks()
with TestRun.step("Stop cache with option '--no-data-flush'."):
cache.stop(no_data_flush=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"Expected caches count: 0; Actual caches count: {caches_count}."
)
with TestRun.step("Try to start cache without loading metadata."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_part.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
cli_messages.check_stderr_msg(
output, cli_messages.start_cache_with_existing_metadata)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1 Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step(
"Compare dirty blocks number before and after loading cache."):
if dirty_blocks_before != core.get_dirty_blocks():
TestRun.LOGGER.error(
"Dirty blocks number is different than before loading cache.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_stop_no_flush_load_cache(cache_mode, filesystem):
"""
title: Test to check that 'stop --no-data-flush' command works correctly.
description: |
Negative test of the ability of CAS to load unflushed cache on core device
with filesystem. Test uses lazy flush cache modes.
pass_criteria:
- No system crash while load cache.
- Starting cache without loading metadata fails.
- Starting cache with loading metadata finishes with success.
"""
with TestRun.step("Prepare cache and core devices."):
cache_part, core_part = prepare()
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Change cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step(
f"Add core with {filesystem.name} filesystem to cache and mount it."
):
core_part.create_filesystem(filesystem)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(
f"Create test file in mount point of exported object and check its md5 sum."
):
test_file = fs_utils.create_random_test_file(test_file_path,
Size(48, Unit.MebiByte))
test_file_md5_before = test_file.md5sum()
with TestRun.step("Unmount exported object."):
core.unmount()
with TestRun.step("Count dirty blocks on exported object."):
dirty_blocks_before = core.get_dirty_blocks()
with TestRun.step("Stop cache with option '--no-data-flush'."):
cache.stop(no_data_flush=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"Expected caches count: 0; Actual caches count: {caches_count}."
)
with TestRun.step("Try to start cache without loading metadata."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_part.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
cli_messages.check_stderr_msg(
output, cli_messages.start_cache_with_existing_metadata)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1 Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step(
"Compare dirty blocks number before and after loading cache."):
if dirty_blocks_before != core.get_dirty_blocks():
TestRun.LOGGER.error(
"Dirty blocks number is different than before loading cache.")
with TestRun.step("Mount exported object."):
core.mount(mount_point)
with TestRun.step(
"Compare md5 sum of test file before and after loading cache."):
if test_file_md5_before != test_file.md5sum():
TestRun.LOGGER.error(
"Test file's md5 sum is different than before loading cache.")
with TestRun.step("Unmount exported object."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_stress_small_cas_device(cache_line_size, cores_number, cache_config):
"""
title: Stress test for verifying data on small CAS devices.
description: |
Validate the ability of CAS to handle many iops when device is small
using different cache modes, cache line sizes and core numbers.
pass_criteria:
- No system crash.
- Md5 sums of core device and exported object are equal.
"""
cache_mode, cleaning_policy = cache_config
with TestRun.step(f"Prepare 1 cache and {cores_number} core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(100, Unit.MebiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_sizes = [Size(200, Unit.MebiByte)] * cores_number
core_dev.create_partitions(core_sizes)
with TestRun.step(f"Start cache with {cores_number} cores."):
cache = casadm.start_cache(cache_part, cache_mode, cache_line_size, force=True)
cores = []
for i in range(cores_number):
cores.append(cache.add_core(core_dev.partitions[i]))
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != cores_number:
TestRun.fail(
f"Expected cores count: {cores_number}; Actual cores count: {cores_count}.")
if cleaning_policy is not None:
with TestRun.step("Set cleaning policy."):
cache.set_cleaning_policy(cleaning_policy)
with TestRun.step(f"Stress cache for {int(stress_time.total_seconds() / 60)} minutes."):
fio = (Fio().create_command()
.io_engine(IoEngine.libaio)
.io_depth(128)
.direct()
.time_based()
.run_time(stress_time)
.read_write(ReadWrite.randrw)
.block_size(cache_line_size)
.num_jobs(cores_number)
.cpus_allowed_policy(CpusAllowedPolicy.split))
for core in cores:
fio.add_job(f"job_{core.core_id}").target(core.system_path)
output = fio.run()[0]
TestRun.LOGGER.info(f"Total read I/O [KiB]: {str(output.read_io())}\n"
f"Total write I/O [KiB]: {str(output.write_io())}")
with TestRun.step("Count md5 sum for exported objects"):
md5sum_core = []
for core in cores:
md5sum_core.append(TestRun.executor.run(
f"md5sum -b {core.system_path}").stdout.split(" ")[0])
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Count md5 sum for core devices"):
md5sum_core_dev = []
for core_dev in core_dev.partitions:
md5sum_core_dev.append(TestRun.executor.run(
f"md5sum -b {core_dev.system_path}").stdout.split(" ")[0])
with TestRun.step("Compare md5 sum of exported objects and cores."):
if md5sum_core_dev != md5sum_core:
TestRun.LOGGER.error(f"Md5 sums of core devices and of exported objects are different.")
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
