def test_stress_load(cache_mode):
"""
title: Stress test for stopping and loading CAS device.
description: |
Validate the ability of the CAS to load and stop cache in the loop
using different cache modes.
pass_criteria:
- No system crash while stop and load cache in the loop.
- CAS device loads successfully.
"""
with TestRun.step("Prepare cache and core."):
cache_dev, core_dev = prepare()
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
casadm.add_core(cache, core_dev)
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Stop cache and load it {iterations_per_config} times."):
with TestRun.step("Stop cache."):
casadm.stop_cache(cache.cache_id)
if len(casadm_parser.get_caches()) != 0:
TestRun.fail("Cache did not stop successfully.")
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Cache did not load successfully - wrong number of caches: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.LOGGER.error(f"Cache loaded with wrong cores count: {cores_count}.")
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_attach_core_to_incomplete_cache_volume():
"""
title: Test for attaching device to inactive cache volume.
description: |
Try to attach core device to inactive cache volume and check if it is visible in OS
properly.
pass_criteria:
- No kernel error
- Core status changes properly
- Cache loads with inactive core device
- Cache status changes properly
- Exported object is present only for active core
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 1)])
cache_dev = devices["cache"].partitions[0]
core_dev = devices["core"].partitions[0]
plug_device = devices["core"]
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, force=True)
core = cache.add_core(core_dev)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
with TestRun.step(
"Check if there is CAS device in /dev and core is in active status."
):
core.check_if_is_present_in_os()
core_status = core.get_status()
if core_status != CoreStatus.active:
TestRun.fail(
f"Core should be in active state. (Actual: {core_status})")
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Unplug core device."):
plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Check if there is no CAS device in /dev and core is in inactive status."
):
core.check_if_is_present_in_os(False)
if core.get_status() != CoreStatus.inactive:
TestRun.fail("Core should be in inactive state.")
with TestRun.step("Plug core device."):
plug_device.plug()
with TestRun.step(
"Check if core status changed to active and CAS device is visible in OS."
):
core.wait_for_status_change(CoreStatus.active)
core.check_if_is_present_in_os()
if cache.get_status() != CacheStatus.running:
TestRun.fail(
"Cache did not change status to 'running' after plugging core device."
)
def test_load_after_clean_shutdown(reboot_type, cache_mode, filesystem):
"""
title: Planned system shutdown test.
description: Test for data consistency after clean system shutdown.
pass_criteria:
- DUT should reboot successfully.
- Checksum of file on core device should be the same before and after reboot.
"""
with TestRun.step("Prepare CAS device."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_dev = TestRun.disks['core']
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
core = cache.add_core(core_dev)
core.create_filesystem(filesystem,
blocksize=int(Size(1, Unit.Blocks4096)))
core.mount(mount_point)
with TestRun.step("Create file on cache and count its checksum."):
test_file = File(os.path.join(mount_point, "test_file"))
Dd()\
.input("/dev/zero")\
.output(test_file.full_path)\
.block_size(Size(1, Unit.KibiByte))\
.count(1024)\
.run()
test_file.refresh_item()
test_file_md5 = test_file.md5sum()
sync()
drop_caches(DropCachesMode.ALL)
with TestRun.step("Reset platform."):
if reboot_type == "soft":
TestRun.executor.reboot()
else:
power_control = TestRun.plugin_manager.get_plugin('power_control')
power_control.power_cycle()
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
core.mount(mount_point)
with TestRun.step("Check file md5sum."):
test_file.refresh_item()
if test_file_md5 != test_file.md5sum():
TestRun.LOGGER.error(
"Checksums does not match - file is corrupted.")
else:
TestRun.LOGGER.info("File checksum is correct.")
with TestRun.step("Remove test file."):
test_file.remove()
def test_load_occupied_id(prepare_and_cleanup):
"""
1. Start new cache instance (don't specify cache id)
2. Add core to newly create cache.
3. Stop cache instance.
4. Start new cache instance on another device (don't specify cache id).
5. Try to load metadata from first device.
* Load should fail.
"""
prepare()
cache_device = next(disk for disk in TestProperties.dut.disks
if disk.disk_type in [DiskType.optane, DiskType.nand])
core_device = next(disk for disk in TestProperties.dut.disks
if (disk.disk_type.value > cache_device.disk_type.value
and disk != cache_device))
TestProperties.LOGGER.info("Creating partitons for test")
cache_device.create_partitions(
[Size(500, Unit.MebiByte),
Size(500, Unit.MebiByte)])
core_device.create_partitions([Size(1, Unit.GibiByte)])
cache_device_1 = cache_device.partitions[0]
cache_device_2 = cache_device.partitions[1]
core_device = core_device.partitions[0]
TestProperties.LOGGER.info("Starting cache with default id and one core")
cache1 = casadm.start_cache(cache_device_1, force=True)
cache1.add_core(core_device)
TestProperties.LOGGER.info("Stopping cache")
cache1.stop()
TestProperties.LOGGER.info(
"Starting cache with default id on different device")
cache2 = casadm.start_cache(cache_device_2, force=True)
TestProperties.LOGGER.info(
"Attempt to load metadata from first cache device")
try:
casadm.load_cache(cache_device_1)
except Exception:
pass
caches = casadm_parser.get_caches()
assert len(caches) == 1, "Inappropirate number of caches after load!"
assert caches[0].cache_device.system_path == cache_device_2.system_path
assert caches[0].cache_id == 1
cores = caches[0].get_core_devices()
assert len(cores) == 0
def test_load_occupied_id():
"""
title: Negative test for loading cache with occupied ID.
description: |
Verify that loading cache with occupied ID is not permitted.
pass_criteria:
- Loading cache with occupied ID should fail.
"""
with TestRun.step("Create partitions for test."):
cache_device = TestRun.disks['cache']
core_device = TestRun.disks['core']
cache_device.create_partitions(
[Size(500, Unit.MebiByte),
Size(500, Unit.MebiByte)])
core_device.create_partitions([Size(1, Unit.GibiByte)])
cache_device_1 = cache_device.partitions[0]
cache_device_2 = cache_device.partitions[1]
core_device = core_device.partitions[0]
with TestRun.step("Start cache with default id and one core."):
cache1 = casadm.start_cache(cache_device_1, force=True)
cache1.add_core(core_device)
with TestRun.step("Stop cache."):
cache1.stop()
with TestRun.step("Start cache with default id on different device."):
casadm.start_cache(cache_device_2, force=True)
with TestRun.step("Attempt to load metadata from first cache device."):
try:
casadm.load_cache(cache_device_1)
TestRun.fail("Cache loaded successfully but it should not.")
except Exception:
pass
caches = casadm_parser.get_caches()
if len(caches) != 1:
TestRun.LOGGER.error("Inappropriate number of caches after load!")
if caches[0].cache_device.path != cache_device_2.path:
TestRun.LOGGER.error("Wrong cache device system path!")
if caches[0].cache_id != 1:
TestRun.LOGGER.error("Wrong cache id.")
cores = caches[0].get_core_devices()
if len(cores) != 0:
TestRun.LOGGER.error("Inappropriate number of cores after load!")
def test_stress_reload_cache(cache_mode):
"""
title: Stress test for reloading cache with simple data integrity check.
description: |
Validate the ability of CAS to reload cache in the loop
with no data corruption.
pass_criteria:
- No system crash while reloading cache.
- CAS device loads successfully.
- No data corruption.
"""
with TestRun.step("Prepare cache and core. Create test file and count it's checksum."):
cache, core, md5_before_load, size_before_load, permissions_before_load = \
prepare_with_file_creation(cache_mode)
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Stop and load cache {iterations_per_config} times."):
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Check md5 of test file."):
check_files(core, size_before_load, permissions_before_load, md5_before_load)
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_seq_cutoff_threshold_load(threshold):
"""
title: Sequential cut-off threshold set/get test with cache load between
description: |
Test if after cache load sequential cut-off threshold
value previously set is being loaded correctly. Each of possible sequential cut-off
policies is set for different core.
pass_criteria:
- Sequential cut-off threshold obtained from get-param command after cache load
must be the same as the one used in set-param command before cache stop
"""
with TestRun.step("Test prepare (start cache and add core)"):
cache, cores = prepare()
_threshold = Size(threshold, Unit.KibiByte)
with TestRun.step(f"Setting cache sequential cut off threshold to "
f"{_threshold}"):
cores[0].set_seq_cutoff_threshold(_threshold)
with TestRun.step("Stopping cache"):
cache.stop()
with TestRun.step("Loading cache"):
loaded_cache = casadm.load_cache(cache.cache_device)
with TestRun.step("Getting core from loaded cache"):
cores_load = loaded_cache.get_core_devices()
with TestRun.step("Check if proper sequential cut off policy was loaded"):
if cores_load[0].get_seq_cut_off_threshold() != _threshold:
TestRun.fail(f"Wrong sequential cut off threshold set: "
f"{cores_load[0].get_seq_cut_off_threshold()} "
f"should be {_threshold}")
def test_dirty_load():
"""
title: Loading cache after dirty shutdown.
description: Test for loading cache containing dirty data after DUT hard restart.
pass_criteria:
- DUT should reboot successfully.
- Cache should load successfully.
"""
with TestRun.step("Prepare devices."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks['core']
core_disk.create_partitions([Size(2, Unit.GibiByte)] * 2)
core_devices = core_disk.partitions
with TestRun.step("Start cache in Write-Back mode and add cores."):
cache = casadm.start_cache(cache_dev, cache_mode=CacheMode.WB)
cores = []
for dev in core_devices:
cores.append(cache.add_core(dev))
with TestRun.step("Set cleaning policy to nop."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Populate cache with dirty data."):
fio = Fio().create_command()\
.size(Size(1, Unit.GibiByte))\
.read_write(ReadWrite.randwrite)\
.io_engine(IoEngine.libaio)\
.block_size(Size(1, Unit.Blocks4096))
for i, core in enumerate(cores):
fio.add_job(f"core{i}").target(core.path)
fio.run()
if cache.get_dirty_blocks() <= Size.zero():
TestRun.fail("Cache does not contain dirty data.")
with TestRun.step("Remove one core without flushing dirty data."):
casadm.remove_core_with_script_command(cache.cache_id, core.core_id,
True)
with TestRun.step("Reset platform."):
power_control = TestRun.plugin_manager.get_plugin('power_control')
power_control.power_cycle()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
caches_num = len(casadm_parser.get_caches())
if caches_num != 1:
TestRun.LOGGER.error(
f"Wrong number of caches. Expected 1, actual {caches_num}.")
cores_num = len(casadm_parser.get_cores(cache.cache_id))
if cores_num != 1:
TestRun.LOGGER.error(
f"Wrong number of cores. Expected 1, actual {cores_num}.")
def test_attach_core_pool():
"""
title: Attaching from core pool on cache load.
description: |
Check that CAS has the ability on cache load to attach core devices that were added to
core device pool if those devices were previously used by cache instance being loaded.
Prevent attaching core device if they were not previously used.
pass_criteria:
- No system crash while reloading CAS modules.
- Core device was added successfully to core pool.
- Core device has been successfully attached to cache on cache load.
- Second core device was not attached to the cache instance.
"""
with TestRun.step("Prepare devices."):
cache_disk = TestRun.disks["cache"]
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks["core"]
core_disk.create_partitions(
[Size(2, Unit.GibiByte),
Size(2, Unit.GibiByte)])
core_dev = core_disk.partitions[0]
second_core_dev = core_disk.partitions[1]
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, force=True)
with TestRun.step("Add core device."):
cache.add_core(core_dev)
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step(
"Add previously used core device to core pool using --try-add flag."
):
first_core = casadm.try_add(core_dev, cache.cache_id)
with TestRun.step(
"Add different core device to core pool using --try-add flag."):
second_core = casadm.try_add(second_core_dev, cache.cache_id)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step("Check each core status."):
if first_core.get_status() is not CoreStatus.active:
TestRun.fail(
f"First core status should be active but is {first_core.get_status()}."
)
if second_core.get_status() is not CoreStatus.detached:
TestRun.fail(
f"Second core status should be detached but is {second_core.get_status()}."
)
with TestRun.step("Stop cache and remove core from core pool."):
casadm.remove_all_detached_cores()
cache.stop()
def test_load_cache_with_mounted_core(cache_mode):
"""
title: Fault injection test for adding mounted core on cache load.
description: |
Negative test of the ability of CAS to add to cache while its loading
core device which is mounted.
pass_criteria:
- No system crash while loading cache.
- Adding mounted core while loading cache fails.
"""
with TestRun.step("Prepare cache and core devices. Start CAS."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_part = core_dev.partitions[0]
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Add core device with xfs filesystem to cache and mount it."):
core_part.create_filesystem(Filesystem.xfs)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(f"Create test file in mount point of exported object and check its md5 sum."):
test_file = fs_utils.create_random_test_file(test_file_path)
test_file_md5_before = test_file.md5sum()
with TestRun.step("Unmount core device."):
core.unmount()
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
with TestRun.step("Mount core device."):
core_part.mount(mount_point)
with TestRun.step("Try to load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1 Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(f"Expected cores count: 0; Actual cores count: {cores_count}.")
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5_before != test_file.md5sum():
TestRun.LOGGER.error("Md5 sum of test file is different.")
core_part.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_seq_cutoff_policy_load():
"""
title: Sequential cut-off policy set/get test with cache load between
description: |
Set each possible policy for different core, stop cache, test if after cache load
sequential cut-off policy value previously set is being loaded correctly for each core.
pass_criteria:
- Sequential cut-off policy obtained from get-param command after cache load
must be the same as the one used in set-param command before cache stop
- Sequential cut-off policy loaded for the last core should be the default one
"""
with TestRun.step(
f"Test prepare (start cache and add {len(SeqCutOffPolicy) + 1} cores)"
):
# Create as many cores as many possible policies including default one
cache, cores = prepare(cores_count=len(SeqCutOffPolicy) + 1)
policies = [policy for policy in SeqCutOffPolicy]
for i, core in TestRun.iteration(
enumerate(cores[:-1]), "Set all possible policies "
"except the default one"):
with TestRun.step(f"Setting cache sequential cut off policy mode to "
f"{policies[i]}"):
cores[i].set_seq_cutoff_policy(policies[i])
with TestRun.step("Stopping cache"):
cache.stop()
with TestRun.step("Loading cache"):
loaded_cache = casadm.load_cache(cache.cache_device)
with TestRun.step("Getting cores from loaded cache"):
cores = loaded_cache.get_core_devices()
for i, core in TestRun.iteration(
enumerate(cores[:-1]), "Check if proper policies have "
"been loaded"):
with TestRun.step(
f"Check if proper sequential cut off policy was loaded"):
if cores[i].get_seq_cut_off_policy() != policies[i]:
TestRun.fail(f"Wrong sequential cut off policy loaded: "
f"{cores[i].get_seq_cut_off_policy()} "
f"should be {policies[i]}")
with TestRun.step(
f"Check if proper (default) sequential cut off policy was loaded for "
f"last core"):
if cores[len(SeqCutOffPolicy)].get_seq_cut_off_policy(
) != SeqCutOffPolicy.DEFAULT:
TestRun.fail(
f"Wrong sequential cut off policy loaded: "
f"{cores[len(SeqCutOffPolicy)].get_seq_cut_off_policy()} "
f"should be {SeqCutOffPolicy.DEFAULT}")
def test_stress_reload_cache(cache_mode):
"""
title: Stress test for reloading cache with simple data integrity check.
description: |
Validate the ability of CAS to reload cache in the loop
with no data corruption.
pass_criteria:
- No system crash while reloading cache.
- CAS device loads successfully.
- No data corruption.
"""
with TestRun.step(
"Prepare cache and core. Create test file and count its checksum."
):
cache, core, file, file_md5sum_before = prepare_with_file_creation(
cache_mode)
for _ in TestRun.iteration(
range(0, iterations_per_config),
f"Stop and load cache {iterations_per_config} times."):
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"Expected caches count: 0; Actual caches count: {caches_count}."
)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}."
)
with TestRun.step("Check md5 sum of test file."):
core.mount(mount_point)
file_md5sum_after = file.md5sum()
if file_md5sum_after != file_md5sum_before:
TestRun.LOGGER.error("Md5 sum of test file is different.")
core.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_stress_reload_module(cache_mode):
"""
title: Stress test for reloading CAS modules.
description: Validate the ability of CAS to reload modules in the loop.
pass_criteria:
- No system crash while reloading CAS modules.
- CAS modules reloads with no errors.
- No data corruption.
"""
with TestRun.step("Prepare cache and core. Create test file and count its checksum."):
cache, core, file, file_md5sum_before = prepare_with_file_creation(cache_mode)
with TestRun.step("Save current cache configuration."):
cache_config = cache.get_cache_config()
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Reload CAS modules and check loaded "
f"cache configuration {iterations_per_config} times."):
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(f"Expected cores count: 0; Actual cores count: {cores_count}.")
with TestRun.step("Reload CAS modules."):
cas_module.reload_all_cas_modules()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Validate cache configuration."):
if cache.get_cache_config() != cache_config:
TestRun.fail("Cache configuration is different than before reloading modules.")
with TestRun.step("Check md5 sum of test file."):
core.mount(mount_point)
file_md5sum_after = file.md5sum()
if file_md5sum_after != file_md5sum_before:
TestRun.LOGGER.error("Md5 sum of test file is different.")
core.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_recovery_flush_reset_fs(cache_mode, fs):
"""
title: Recovery after reset during cache flushing - test on filesystem.
description: |
Verify that unflushed data can be safely recovered, when reset was pressed during
data flushing on filesystem.
pass_criteria:
- CAS recovers successfully after reboot
- No data corruption
"""
with TestRun.step("Prepare cache and core devices."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(16, Unit.GibiByte)] * 2)
cache_device = cache_disk.partitions[0]
core_device = core_disk.partitions[0]
core_device_link = core_device.get_device_link("/dev/disk/by-id")
cache_device_link = cache_device.get_device_link("/dev/disk/by-id")
with TestRun.step(f"Create {fs} filesystem on core."):
core_device.create_filesystem(fs)
with TestRun.step("Create test files."):
source_file, target_file = create_test_files(test_file_size)
with TestRun.step("Setup cache and add core."):
cache = casadm.start_cache(cache_device, cache_mode)
Udev.disable()
core = cache.add_core(core_device)
cache.set_cleaning_policy(CleaningPolicy.nop)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Mount CAS device."):
core.mount(mount_point)
with TestRun.step("Copy file to CAS."):
copy_file(source=source_file.full_path,
target=os.path.join(mount_point, "source_test_file"),
size=test_file_size,
direct="oflag")
with TestRun.step("Unmount CAS device."):
core.unmount()
with TestRun.step("Trigger flush."):
TestRun.executor.run_in_background(
cli.flush_cache_cmd(f"{cache.cache_id}"))
with TestRun.step("Hard reset DUT during data flushing."):
power_cycle_dut(True, core_device)
cache_device.path = cache_device_link.get_target()
core_device.path = core_device_link.get_target()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_device)
if cache.get_dirty_blocks() == Size.zero():
TestRun.fail("There are no dirty blocks on cache device.")
with TestRun.step("Stop cache with dirty data flush."):
core_writes_before = core_device.get_io_stats().sectors_written
cache.stop()
if core_writes_before >= core_device.get_io_stats().sectors_written:
TestRun.fail(
"No data was flushed after stopping cache started with load option."
)
with TestRun.step("Mount core device."):
core_device.mount(mount_point)
with TestRun.step(
"Copy test file from core device to temporary location. "
"Compare it with the first version – they should be the same."):
copy_file(source=os.path.join(mount_point, "source_test_file"),
target=target_file.full_path,
size=test_file_size,
direct="iflag")
compare_files(source_file, target_file)
with TestRun.step("Unmount core device and remove test files."):
core_device.unmount()
target_file.remove()
source_file.remove()
Udev.enable()
def test_preserve_data_for_inactive_device():
"""
title: Validate preserving data for inactive CAS devices.
description: Validate that cached data for inactive CAS devices is preserved.
pass_criteria:
- No kernel error
- File md5 checksums match in every iteration.
- Cache read hits increase after reads (md5 checksum) from CAS device with attached core.
"""
mount_dir = "/mnt/test"
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 1)])
cache_dev = devices["cache"].partitions[0]
core_dev = devices["core"].partitions[0]
plug_device = devices["core"]
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
cache.set_cleaning_policy(CleaningPolicy.nop)
core = cache.add_core(core_dev)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Create filesystem on CAS device and mount it."):
core.create_filesystem(Filesystem.ext3)
core.mount(mount_dir)
with TestRun.step(
"Create a test file with random writes on mount point and count it's md5."
):
file_path = f"{mount_dir}/test_file"
test_file = File.create_file(file_path)
dd = Dd().input("/dev/random") \
.output(file_path) \
.count(100) \
.block_size(Size(1, Unit.Blocks512))
dd.run()
os_utils.sync()
md5_after_create = test_file.md5sum()
cache_stats_before_stop = cache.get_statistics()
core_stats_before_stop = core.get_statistics()
with TestRun.step("Unmount CAS device."):
core.unmount()
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug core device."):
plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
cache_stats_after_load = cache.get_statistics()
core_stats_after_load = core.get_statistics()
if cache_stats_before_stop.usage_stats.clean != cache_stats_after_load.usage_stats.clean or\
cache_stats_before_stop.usage_stats.dirty != \
cache_stats_after_load.usage_stats.dirty or\
core_stats_before_stop.usage_stats.clean != \
core_stats_after_load.usage_stats.clean or\
core_stats_before_stop.usage_stats.dirty != core_stats_after_load.usage_stats.dirty:
TestRun.fail(
f"Statistics after counting md5 are different than after cache load.\n"
f"Cache stats before: {cache_stats_before_stop}\n"
f"Cache stats after: {cache_stats_after_load}\n"
f"Core stats before: {core_stats_before_stop}\n"
f"Core stats after: {core_stats_after_load}")
with TestRun.step(
"Plug core disk using sysfs and verify this change is reflected "
"on the cache list."):
plug_device.plug()
if cache.get_status() != CacheStatus.running or core.get_status(
) != CoreStatus.active:
TestRun.fail(
f"Expected cache status is running (actual - {cache.get_status()}).\n"
f"Expected core status is active (actual - {core.get_status()})."
)
with TestRun.step("Mount CAS device"):
core.mount(mount_dir)
with TestRun.step(
"Count md5 checksum for test file and compare it with previous value."
):
cache_read_hits_before_md5 = cache.get_statistics(
).request_stats.read.hits
md5_after_cache_load = test_file.md5sum()
if md5_after_create != md5_after_cache_load:
TestRun.fail(
"Md5 checksum after cache load operation is different than before "
"stopping cache.")
else:
TestRun.LOGGER.info(
"Md5 checksum is identical before and after cache load operation "
"with inactive CAS device.")
with TestRun.step(
"Verify that cache read hits increased after counting md5 checksum."
):
cache_read_hits_after_md5 = cache.get_statistics(
).request_stats.read.hits
if cache_read_hits_after_md5 - cache_read_hits_before_md5 < 0:
TestRun.fail(
f"Cache read hits did not increase after counting md5 checksum. "
f"Before: {cache_read_hits_before_md5}. "
f"After: {cache_read_hits_after_md5}.")
else:
TestRun.LOGGER.info("Cache read hits increased as expected.")
with TestRun.step("Unmount CAS device and stop cache."):
core.unmount()
cache.stop()
def test_print_statistics_inactive(cache_mode):
"""
title: Print statistics for cache with inactive cache volumes.
description: |
Check if statistics are displayed properly when there is one or more
inactive cache volumes added to cache.
pass_criteria:
- No kernel error
- All statistics should contain appropriate information depending on situation of
cache and core devices (as described in test steps)
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core1", 1), ("core2", 1)])
cache_dev = devices["cache"].partitions[0]
first_core_dev = devices["core1"].partitions[0]
second_core_dev = devices["core2"].partitions[0]
first_plug_device = devices["core1"]
second_plug_device = devices["core2"]
with TestRun.step("Start cache and add cores."):
cache = casadm.start_cache(cache_dev,
cache_mode=cache_mode,
force=True)
first_core = cache.add_core(first_core_dev)
second_core = cache.add_core(second_core_dev)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run IO."):
run_fio([first_core.system_path, second_core.system_path])
with TestRun.step(
"Print statistics and check if there is no inactive usage section."
):
active_stats = cache.get_statistics()
check_if_inactive_section_exists(active_stats, False)
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Remove both core devices from OS."):
first_plug_device.unplug()
second_plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Check if inactive devices section appeared and contains appropriate "
"information."):
inactive_stats_before = cache.get_statistics()
check_if_inactive_section_exists(inactive_stats_before)
check_number_of_inactive_devices(inactive_stats_before, 2)
with TestRun.step(
"Attach one of detached core devices and add it to cache."):
first_plug_device.plug()
first_core_status = first_core.get_status()
if first_core_status != CoreStatus.active:
TestRun.fail(
f"Core {first_core.system_path} should be in active state but it is not. "
f"Actual state: {first_core_status}.")
with TestRun.step("Check cache statistics section of inactive devices."):
inactive_stats_after = cache.get_statistics()
check_if_inactive_section_exists(inactive_stats_after)
check_number_of_inactive_devices(inactive_stats_after, 1)
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_occupancy,
inactive_stats_after.inactive_usage_stats.inactive_occupancy,
"inactive occupancy",
cache.get_cache_mode() == CacheMode.PT)
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_clean,
inactive_stats_after.inactive_usage_stats.inactive_clean,
"inactive clean",
cache.get_cache_mode() in [CacheMode.PT, CacheMode.WB])
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_dirty,
inactive_stats_after.inactive_usage_stats.inactive_dirty,
"inactive dirty",
cache.get_cache_mode() != CacheMode.WB)
with TestRun.step("Check statistics per inactive core."):
inactive_core_stats = second_core.get_statistics()
if inactive_stats_after.inactive_usage_stats.inactive_occupancy == \
inactive_core_stats.usage_stats.occupancy:
TestRun.LOGGER.info(
"Inactive occupancy in cache statistics is equal to inactive core "
"occupancy.")
else:
TestRun.fail(
f"Inactive core occupancy ({inactive_core_stats.usage_stats.occupancy}) "
f"should be the same as cache inactive occupancy "
f"({inactive_stats_after.inactive_usage_stats.inactive_occupancy})."
)
with TestRun.step(
"Remove inactive core from cache and check if cache is in running state."
):
cache.remove_core(second_core.core_id, force=True)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache did not change status to 'running' after plugging core device. "
f"Actual status: {cache_status}.")
with TestRun.step(
"Check if there is no inactive devices statistics section and if cache has "
"Running status."):
cache_stats = cache.get_statistics()
check_if_inactive_section_exists(cache_stats, False)
check_number_of_inactive_devices(cache_stats, 0)
with TestRun.step("Plug missing disk and stop cache."):
second_plug_device.plug()
cache.stop()
def test_recovery_unplug_cache_fs(cache_mode, cls, filesystem, direct):
"""
title: Test for recovery after cache drive removal - test with filesystem.
description: |
Verify that unflushed data can be safely recovered after, when SSD drive is removed
after write completion - test with filesystem.
pass_criteria:
- CAS recovers successfully after cache drive unplug
- No data corruption
"""
with TestRun.step("Prepare devices"):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(16, Unit.GibiByte)])
cache_device = cache_disk.partitions[0]
core_device = core_disk.partitions[0]
with TestRun.step("Create test files."):
source_file, target_file = create_test_files(test_file_size)
with TestRun.step("Create filesystem on core device."):
core_device.create_filesystem(filesystem)
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_device, cache_mode, cls)
core = cache.add_core(core_device)
with TestRun.step("Mount CAS device."):
core.mount(mount_point)
with TestRun.step("Copy file to CAS."):
copy_file(source=source_file.full_path,
target=test_file_path,
size=test_file_size,
direct="oflag" if direct else None)
TestRun.LOGGER.info(str(core.get_statistics()))
with TestRun.step("Unmount CAS device."):
core.unmount()
with TestRun.step("Unplug cache device."):
cache_disk.unplug()
TestRun.LOGGER.info(f"List caches:\n{casadm.list_caches().stdout}")
TestRun.LOGGER.info(
f"Dirty blocks on cache: "
f"{cache.get_dirty_blocks().get_value(Unit.Blocks4096)}")
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Plug missing cache device."):
cache_disk.plug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_device)
TestRun.LOGGER.info(
f"Dirty blocks on cache: "
f"{cache.get_dirty_blocks().get_value(Unit.Blocks4096)}")
with TestRun.step("Stop cache with data flush."):
cache.stop()
with TestRun.step("Mount core device."):
core_device.mount(mount_point)
with TestRun.step("Copy file from core device and check md5sum."):
copy_file(source=test_file_path,
target=target_file.full_path,
size=test_file_size,
direct="iflag" if direct else None)
compare_files(source_file, target_file)
with TestRun.step("Unmount core device and remove files."):
core_device.unmount()
target_file.remove()
source_file.remove()
def test_recovery_all_options(cache_mode, cache_line_size, cleaning_policy,
filesystem):
"""
title: Test for recovery after reset with various cache options.
description: Verify that unflushed data can be safely recovered after reset.
pass_criteria:
- CAS recovers successfully after reboot
- No data corruption
"""
with TestRun.step("Prepare cache and core devices."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(200, Unit.MebiByte)])
core_disk.create_partitions([Size(2000, Unit.MebiByte)] * 2)
cache_device = cache_disk.partitions[0]
core_device = core_disk.partitions[0]
test_file = File(os.path.join(mount_point, filename))
file_operation(test_file.full_path, pattern, ReadWrite.write)
file_md5 = test_file.md5sum()
with TestRun.step(f"Make {filesystem} on core device."):
core_device.create_filesystem(filesystem)
with TestRun.step("Mount core device."):
core_device.mount(mount_point)
file_operation(test_file.full_path, other_pattern, ReadWrite.write)
os_utils.drop_caches(DropCachesMode.ALL)
with TestRun.step("Unmount core device."):
core_device.unmount()
with TestRun.step(
f"Start cache in {cache_mode.name} with given configuration."):
cache = casadm.start_cache(cache_device,
cache_mode,
cache_line_size,
force=True)
cache.set_cleaning_policy(cleaning_policy)
if cleaning_policy == CleaningPolicy.acp:
cache.set_params_acp(
FlushParametersAcp(wake_up_time=Time(seconds=1)))
with TestRun.step("Add core."):
core = cache.add_core(core_device)
with TestRun.step("Mount CAS device."):
core.mount(mount_point)
file_operation(test_file.full_path, pattern, ReadWrite.write)
with TestRun.step(
"Change cache mode to Write-Through without flush option."):
cache.set_cache_mode(CacheMode.WT, flush=False)
with TestRun.step("Reset platform."):
os_utils.sync()
core.unmount()
TestRun.LOGGER.info(
f"Number of dirty blocks in cache: {cache.get_dirty_blocks()}")
power_cycle_dut()
with TestRun.step("Try to start cache without load and force option."):
try:
casadm.start_cache(cache_device, cache_mode, cache_line_size)
TestRun.fail("Cache started without load or force option.")
except Exception:
TestRun.LOGGER.info(
"Cache did not start without load and force option.")
with TestRun.step("Load cache and stop it with flush."):
cache = casadm.load_cache(cache_device)
cache.stop()
with TestRun.step("Check md5sum of tested file on core device."):
core_device.mount(mount_point)
cas_md5 = test_file.md5sum()
core_device.unmount()
if cas_md5 == file_md5:
TestRun.LOGGER.info(
"Source and target file checksums are identical.")
else:
TestRun.fail("Source and target file checksums are different.")
def test_remove_inactive_devices():
"""
title: Validate removing inactive CAS devices.
description: |
Validate that it is possible to remove inactive CAS devices when there are no dirty
cache lines associated with them and that removing CAS devices is prevented otherwise
(unless ‘force’ option is used).
pass_criteria:
- No kernel error
- Removing CAS devices without dirty data is successful.
- Removing CAS devices with dirty data without ‘force’ option is blocked.
- Removing CAS devices with dirty data with ‘force’ option is successful.
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 4)])
cache_dev = devices["cache"].partitions[0]
core_devs = devices["core"].partitions
plug_device = devices["core"]
with TestRun.step("Start cache and add four cores."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cores = []
for d in core_devs:
cores.append(cache.add_core(d))
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run random writes to all CAS devices."):
run_fio([c.system_path for c in cores])
with TestRun.step(
"Flush dirty data from two CAS devices and verify than other two "
"contain dirty data."):
for core in cores:
if core.core_id % 2 == 0:
core.flush_core()
if core.get_dirty_blocks() != Size.zero():
TestRun.fail("Failed to flush CAS device.")
elif core.get_dirty_blocks() == Size.zero():
TestRun.fail("There should be dirty data on CAS device.")
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug core disk."):
plug_device.unplug()
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
with TestRun.step(
"Verify that all previously created CAS devices are listed with "
"proper status."):
for core in cores:
if core.get_status() != CoreStatus.inactive:
TestRun.fail(f"Each core should be in inactive state. "
f"Actual states:\n{casadm.list_caches().stdout}")
with TestRun.step(
"Try removing CAS device without ‘force’ option. Verify that for "
"dirty CAS devices operation is blocked, proper message is displayed "
"and device is still listed."):
shuffle(cores)
for core in cores:
try:
dirty_blocks = core.get_dirty_blocks()
core.remove_core()
if dirty_blocks != Size.zero():
TestRun.fail(
"Removing dirty CAS device should be impossible but remove "
"command executed without any error.")
TestRun.LOGGER.info(
"Removing core with force option skipped for clean CAS device."
)
except CmdException as e:
if dirty_blocks == Size.zero():
TestRun.fail(
"Removing clean CAS device should be possible but remove "
"command returned an error.")
TestRun.LOGGER.info(
"Remove operation without force option is blocked for "
"dirty CAS device as expected.")
cli_messages.check_stderr_msg(
e.output, cli_messages.remove_inactive_core)
output = casadm.list_caches().stdout
if core.system_path not in output:
TestRun.fail(
f"CAS device is not listed in casadm list output but it should be."
f"\n{output}")
core.remove_core(force=True)
with TestRun.step("Plug missing disk and stop cache."):
plug_device.plug()
casadm.stop_all_caches()
def test_stop_cache_with_inactive_devices():
"""
title: Validate stopping cache with inactive CAS devices.
description: |
Validate that cache with inactive CAS devices cannot be stopped
unless ‘force’ option is used.
pass_criteria:
- No kernel error
- Stopping cache with inactive CAS devices without ‘force’ option is blocked.
- Stopping cache with inactive CAS devices with ‘force’ option is successful.
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 1)])
cache_dev = devices["cache"].partitions[0]
core_dev = devices["core"].partitions[0]
plug_device = devices["core"]
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
core = cache.add_core(core_dev)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step(
"Run random writes and verify that CAS device contains dirty data."
):
run_fio([core.system_path])
if core.get_dirty_blocks() == Size.zero():
TestRun.fail("There is no dirty data on core device.")
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug core disk."):
plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Verify that previously created CAS device is listed with proper status."
):
core_status = core.get_status()
if core_status != CoreStatus.inactive:
TestRun.fail(
f"CAS device should be in inactive state. Actual status: {core_status}."
)
with TestRun.step(
"Try stopping cache without ‘no data flush’ option, verify that operation "
"was blocked and proper message is displayed."):
try_stop_incomplete_cache(cache)
with TestRun.step("Stop cache with force option."):
cache.stop(no_data_flush=True)
with TestRun.step("Plug missing core device."):
plug_device.plug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step("Stop cache with flushing dirty data."):
cache.stop()
with TestRun.step("Unplug core device."):
plug_device.unplug()
with TestRun.step("Load cache and verify core status is inactive."):
cache = casadm.load_cache(cache_dev)
core_status = core.get_status()
if core_status != CoreStatus.inactive:
TestRun.fail(
f"CAS device should be in inactive state. Actual state: {core_status}."
)
with TestRun.step(
"Try stopping cache without ‘no data flush’ option, verify that "
"operation was blocked and proper message is displayed."):
try_stop_incomplete_cache(cache)
with TestRun.step(
"Stop cache with 'no data flush' option and plug missing core device."
):
cache.stop(no_data_flush=True)
plug_device.plug()
def test_stop_no_flush_load_cache(cache_mode, filesystem):
"""
title: Test to check that 'stop --no-data-flush' command works correctly.
description: |
Negative test of the ability of CAS to load unflushed cache on core device
with filesystem. Test uses lazy flush cache modes.
pass_criteria:
- No system crash while load cache.
- Starting cache without loading metadata fails.
- Starting cache with loading metadata finishes with success.
"""
with TestRun.step("Prepare cache and core devices."):
cache_part, core_part = prepare()
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Change cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step(
f"Add core with {filesystem.name} filesystem to cache and mount it."
):
core_part.create_filesystem(filesystem)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(
f"Create test file in mount point of exported object and check its md5 sum."
):
test_file = fs_utils.create_random_test_file(test_file_path,
Size(48, Unit.MebiByte))
test_file_md5_before = test_file.md5sum()
with TestRun.step("Unmount exported object."):
core.unmount()
with TestRun.step("Count dirty blocks on exported object."):
dirty_blocks_before = core.get_dirty_blocks()
with TestRun.step("Stop cache with option '--no-data-flush'."):
cache.stop(no_data_flush=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"Expected caches count: 0; Actual caches count: {caches_count}."
)
with TestRun.step("Try to start cache without loading metadata."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_part.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
cli_messages.check_stderr_msg(
output, cli_messages.start_cache_with_existing_metadata)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1 Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step(
"Compare dirty blocks number before and after loading cache."):
if dirty_blocks_before != core.get_dirty_blocks():
TestRun.LOGGER.error(
"Dirty blocks number is different than before loading cache.")
with TestRun.step("Mount exported object."):
core.mount(mount_point)
with TestRun.step(
"Compare md5 sum of test file before and after loading cache."):
if test_file_md5_before != test_file.md5sum():
TestRun.LOGGER.error(
"Test file's md5 sum is different than before loading cache.")
with TestRun.step("Unmount exported object."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_load_x_to_one_without_params(cache_mode, cleaning_policy,
cache_line_size, cores_amount):
"""
title: Test for loading CAS with 1 cache and 1 or 4 cores without extra params.
description: |
Verify that loading cache configurations works properly in every mode
with 1 cache and 1 or 4 cores. Use minimal number of possible parameters
with load command.
pass_criteria:
- Cache loads successfully.
- No errors in cache are found.
"""
with TestRun.step(f"Prepare 1 cache and {cores_amount} core devices"):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_size = []
for i in range(cores_amount):
core_size.append(Size(1, Unit.GibiByte))
core_dev.create_partitions(core_size)
with TestRun.step(f"Start cache with {cores_amount} cores."):
cache = casadm.start_cache(cache_dev,
cache_mode,
cache_line_size,
force=True)
cores = []
for i in range(cores_amount):
cores.append(cache.add_core(core_dev.partitions[i]))
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != cores_amount:
TestRun.fail(
f"Expected cores count: {cores_amount}; Actual cores count: {cores_count}."
)
with TestRun.step("Configure cleaning policy."):
cache.set_cleaning_policy(cleaning_policy)
if cleaning_policy == CleaningPolicy.alru:
alru = FlushParametersAlru()
alru.activity_threshold = Time(milliseconds=1000)
alru.flush_max_buffers = 10
alru.staleness_time = Time(seconds=60)
alru.wake_up_time = Time(seconds=5)
cache.set_params_alru(alru)
if cleaning_policy == CleaningPolicy.acp:
acp = FlushParametersAcp()
acp.flush_max_buffers = 100
acp.wake_up_time = Time(seconds=5)
cache.set_params_acp(acp)
with TestRun.step("Run FIO on exported object"):
fio = (Fio().create_command().io_engine(
IoEngine.libaio).io_depth(64).direct().read_write(
ReadWrite.randrw).size(Size(
1, Unit.GibiByte)).block_size(cache_line_size).read_write(
ReadWrite.randrw).num_jobs(
cores_amount).cpus_allowed_policy(
CpusAllowedPolicy.split))
for core in cores:
fio.add_job(f"job_{core.core_id}").target(core.path)
fio.run()
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"There are still {caches_count} caches running after stopping service."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(
f"There are still {cores_count} cores running after stopping service."
)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != cores_amount:
TestRun.fail(
f"Expected cores count: {cores_amount}; Actual cores count: {cores_count}."
)
with TestRun.step("Compare cache configuration before and after load."):
if cache_mode != cache.get_cache_mode():
TestRun.fail("Cache modes are different.")
if cache_line_size != cache.get_cache_line_size():
TestRun.fail("Cache line sizes are different.")
if cleaning_policy != cache.get_cleaning_policy():
TestRun.fail("Cleaning policies are different.")
if cleaning_policy == CleaningPolicy.alru:
if alru != cache.get_flush_parameters_alru():
TestRun.fail("Cleaning policy parameters are different.")
if cleaning_policy == CleaningPolicy.acp:
if acp != cache.get_flush_parameters_acp():
TestRun.fail("Cleaning policy parameters are different.")
with TestRun.step("Run FIO again on exported object"):
fio = (Fio().create_command().io_engine(
IoEngine.libaio).io_depth(64).direct().read_write(
ReadWrite.randrw).size(Size(
1, Unit.GibiByte)).block_size(cache_line_size).read_write(
ReadWrite.randrw).num_jobs(
cores_amount).cpus_allowed_policy(
CpusAllowedPolicy.split))
for core in cores:
fio.add_job(f"job_{core.core_id}").target(core.path)
fio.run()
with TestRun.step("Check if there are no error statistics."):
if cache.get_statistics().error_stats.total_errors != 0:
TestRun.fail("There are errors in the cache.")
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
def test_recovery_flush_reset_raw(cache_mode):
"""
title: Recovery after reset during cache flushing - test on raw device.
description: |
Verify that unflushed data can be safely recovered, when reset was pressed during
data flushing on raw device.
pass_criteria:
- CAS recovers successfully after reboot
- No data corruption
"""
with TestRun.step("Prepare cache and core devices."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(16, Unit.GibiByte)] * 2)
cache_device = cache_disk.partitions[0]
core_device = core_disk.partitions[0]
core_device_link = core_device.get_device_link("/dev/disk/by-id")
cache_device_link = cache_device.get_device_link("/dev/disk/by-id")
with TestRun.step("Create test files."):
source_file, target_file = create_test_files(test_file_size)
with TestRun.step("Setup cache and add core."):
cache = casadm.start_cache(cache_device, cache_mode)
core = cache.add_core(core_device)
cache.set_cleaning_policy(CleaningPolicy.nop)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Copy file to CAS."):
copy_file(source=source_file.full_path,
target=core.path,
size=test_file_size,
direct="oflag")
with TestRun.step("Sync and flush buffers."):
os_utils.sync()
output = TestRun.executor.run(f"hdparm -f {core.path}")
if output.exit_code != 0:
raise CmdException("Error during hdparm", output)
with TestRun.step("Trigger flush."):
TestRun.executor.run_in_background(
cli.flush_cache_cmd(f"{cache.cache_id}"))
with TestRun.step("Hard reset DUT during data flushing."):
power_cycle_dut(wait_for_flush_begin=True, core_device=core_device)
cache_device.path = cache_device_link.get_target()
core_device.path = core_device_link.get_target()
with TestRun.step(
"Copy file from core and check if current md5sum is different than "
"before restart."):
copy_file(source=core_device_link.get_target(),
target=target_file.full_path,
size=test_file_size,
direct="iflag")
compare_files(source_file, target_file, should_differ=True)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_device)
if cache.get_dirty_blocks() == Size.zero():
TestRun.fail("There are no dirty blocks on cache device.")
with TestRun.step("Stop cache with dirty data flush."):
core_writes_before = core_device.get_io_stats().sectors_written
cache.stop()
if core_writes_before >= core_device.get_io_stats().sectors_written:
TestRun.fail(
"No data was flushed after stopping cache started with load option."
)
with TestRun.step(
"Copy test file from core device to temporary location. "
"Compare it with the first version – they should be the same."):
copy_file(source=core_device_link.get_target(),
target=target_file.full_path,
size=test_file_size,
direct="iflag")
compare_files(source_file, target_file)
with TestRun.step("Cleanup core device and remove test files."):
target_file.remove()
source_file.remove()
def test_user_cli():
"""
title: Test that OpenCAS does not allow to change parameters in CLI by non-root user.
description: |
Checking if changing parameters in CLI by non-root user is forbidden by OpenCAS,
but is permitted with 'sudo' command.
pass_criteria:
- Non-root user can only print help and CAS version.
- Sudoer user is allowed to change OpenCAS parameters in CLI with sudo.
"""
with TestRun.step("Prepare cache and core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(256, Unit.MebiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte), Size(256, Unit.MebiByte)])
core_part1 = core_dev.partitions[0]
core_part2 = core_dev.partitions[1]
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, force=True)
with TestRun.step("Add core to cache and mount it."):
core_part1.create_filesystem(Filesystem.ext3)
core = cache.add_core(core_part1)
core.mount(mount_point)
with TestRun.step(f"Copy casadm bin from {system_casadm_bin_path} "
f"to {user_casadm_bin_dest_path}."):
casadm_bin = fs_utils.parse_ls_output(fs_utils.ls_item(f"{system_casadm_bin_path}"))[0]
casadm_bin_copy = casadm_bin.copy(user_casadm_bin_dest_path, True)
casadm_bin_copy.chmod_numerical(777)
with TestRun.step("Copy IO class config."):
io_conf = fs_utils.parse_ls_output(fs_utils.ls_item(f"{ioclass_config_path}"))[0]
io_conf_copy = io_conf.copy(ioclass_config_copy_path, force=True)
with TestRun.step("Unmount core."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
with TestRun.step("Add non-root user account."):
TestRun.executor.run(f"useradd -N -r -l {user_name}")
user_home_dir = fs_utils.parse_ls_output(fs_utils.ls_item(f"/home/{user_name}"))[0]
user_home_dir.chmod_numerical(777, True)
with TestRun.step("Try to start cache."):
try:
output = run_as_other_user(cli.start_cmd(cache_dev.path), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Starting cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot start cache.")
with TestRun.step("Start cache again."):
casadm.load_cache(cache_dev)
with TestRun.step("Try to stop cache."):
try:
output = run_as_other_user(cli.stop_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Stopping cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot stop cache.")
with TestRun.step("Try to set cache mode."):
try:
output = run_as_other_user(cli.set_cache_mode_cmd(CacheMode.WB,
str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cache mode should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cache mode.")
with TestRun.step("Try to add core to cache."):
try:
output = run_as_other_user(cli.add_core_cmd(str(cache.cache_id),
core_part2.path), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Adding core to cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot add core.")
with TestRun.step("Try to remove core from cache."):
try:
output = run_as_other_user(cli.remove_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Removing core from cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot remove core.")
with TestRun.step("Try to zero metadata."):
try:
output = run_as_other_user(cli.zero_metadata_cmd(str(cache_dev)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Zeroing metadata should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot zero metadata.")
with TestRun.step("Try to list caches."):
try:
output = run_as_other_user(cli.list_cmd(), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Listing caches should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot list caches.")
with TestRun.step("Try to print stats."):
try:
output = run_as_other_user(cli.print_statistics_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Printing stats should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot print statistics.")
with TestRun.step("Try to reset stats."):
try:
output = run_as_other_user(cli.reset_counters_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Resetting stats should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot reset statistics.")
with TestRun.step("Try to flush cache."):
try:
output = run_as_other_user(cli.flush_cache_cmd(str(cache.cache_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Flushing cache should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot flush cache.")
with TestRun.step("Try to flush core."):
try:
output = run_as_other_user(cli.flush_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Flushing core should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot flush core.")
with TestRun.step("Try to set cleaning policy and its parameters."):
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "nop"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as nop.")
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "alru"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as alru.")
try:
output = run_as_other_user(cli.set_param_cleaning_alru_cmd(str(cache.cache_id),
"15",
"60",
"1000",
"8000"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy parameters should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set alru cleaning policy parameters.")
try:
output = run_as_other_user(cli.set_param_cleaning_cmd(
str(cache.cache_id), "acp"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set cleaning policy as acp.")
try:
output = run_as_other_user(cli.set_param_cleaning_acp_cmd(str(cache.cache_id),
"15",
"1000"), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Setting cleaning policy parameters should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot set acp cleaning policy parameters.")
with TestRun.step("Try to list IO class configuration."):
try:
output = run_as_other_user(cli.list_io_classes_cmd(
str(cache.cache_id), OutputFormat.table.name), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Listing IO class configuration should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot list IO class configuration.")
with TestRun.step("Try to load IO class configuration."):
try:
output = run_as_other_user(cli.load_io_classes_cmd(
str(cache.cache_id), io_conf_copy), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Loading IO class configuration should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot load IO class configuration.")
with TestRun.step("Try to print help for casadm."):
try:
run_as_other_user(cli.help_cmd(), user_name)
except CmdException:
TestRun.LOGGER.error("Non-root user should be able to print help for casadm.")
with TestRun.step("Try to print version of OpenCAS."):
try:
run_as_other_user(cli.version_cmd(), user_name)
except CmdException:
TestRun.LOGGER.error("Non-root user should be able to print version of OpenCAS.")
with TestRun.step("Add non-root user account to sudoers group."):
TestRun.executor.run(f'echo "{user_name} ALL = (root) NOPASSWD:ALL" '
f'| sudo tee /etc/sudoers.d/{user_name}')
with TestRun.step("Try to stop cache with 'sudo'."):
try:
run_as_other_user(cli.stop_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to stop cache.")
with TestRun.step("Try to start cache with 'sudo'."):
try:
run_as_other_user(cli.start_cmd(cache_dev.path, force=True), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to start cache.")
with TestRun.step("Try to set cache mode with 'sudo'."):
try:
run_as_other_user(
cli.set_cache_mode_cmd(str(CacheMode.WB.name).lower(), str(cache.cache_id)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to set cache mode.")
with TestRun.step("Try to add core to cache with 'sudo'."):
try:
run_as_other_user(cli.add_core_cmd(str(cache.cache_id),
core_part1.path), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to add core to cache.")
with TestRun.step("Try to list caches with 'sudo'."):
try:
run_as_other_user(cli.list_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to list caches.")
with TestRun.step("Try to print stats with 'sudo'."):
try:
run_as_other_user(cli.print_statistics_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print stats.")
with TestRun.step("Try to reset stats with 'sudo'."):
try:
run_as_other_user(cli.reset_counters_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to reset stats.")
with TestRun.step("Try to flush cache with 'sudo'."):
try:
run_as_other_user(cli.flush_cache_cmd(str(cache.cache_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to flush cache.")
with TestRun.step("Try to flush core with 'sudo'."):
try:
run_as_other_user(cli.flush_core_cmd(str(cache.cache_id),
str(core.core_id)), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to flush core.")
with TestRun.step("Try to set cleaning policy and its parameters with 'sudo'."):
try:
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "nop"),
user_name, True)
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "alru"),
user_name, True)
try:
run_as_other_user(cli.set_param_cleaning_alru_cmd(str(cache.cache_id),
"15",
"60",
"1000",
"8000"), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set alru cleaning policy parameters.")
run_as_other_user(cli.set_param_cleaning_cmd(str(cache.cache_id), "acp"),
user_name, True)
try:
run_as_other_user(cli.set_param_cleaning_acp_cmd(str(cache.cache_id),
"15",
"1000"), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set acp cleaning policy parameters.")
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"set cleaning policy and its parameters.")
with TestRun.step("Try to list IO class with 'sudo'."):
try:
run_as_other_user(cli.list_io_classes_cmd(str(cache.cache_id), OutputFormat.table.name),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to list IO class.")
with TestRun.step("Try to load IO class configuration with 'sudo'."):
try:
run_as_other_user(cli.load_io_classes_cmd(str(cache.cache_id), io_conf_copy),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to "
"load IO class configuration.")
with TestRun.step("Try to remove core from cache with 'sudo'."):
try:
run_as_other_user(cli.remove_core_cmd(str(cache.cache_id), str(core.core_id)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to remove core from cache.")
with TestRun.step("Try to zero metadata with 'sudo'."):
try:
run_as_other_user(cli.zero_metadata_cmd(str(cache_dev)),
user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to zero metadata.")
with TestRun.step("Try to print help for casadm with 'sudo'."):
try:
run_as_other_user(cli.help_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print help for casadm.")
with TestRun.step("Try to print version of OpenCAS with 'sudo'."):
try:
run_as_other_user(cli.version_cmd(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to print version of OpenCAS.")
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
with TestRun.step("Remove user account."):
TestRun.executor.run(f"userdel -r -Z {user_name}")
def test_stop_no_flush_load_cache_no_fs(cache_mode):
"""
title: Test to check that 'stop --no-data-flush' command works correctly.
description: |
Negative test of the ability of CAS to load unflushed cache on core device
without filesystem. Test uses lazy flush cache modes.
pass_criteria:
- No system crash while load cache.
- Starting cache without loading metadata fails.
- Starting cache with loading metadata finishes with success.
"""
with TestRun.step("Prepare cache and core devices."):
cache_part, core_part = prepare()
with TestRun.step("Start cache with --force option."):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Change cleaning policy to NOP."):
cache.set_cleaning_policy(CleaningPolicy.nop)
with TestRun.step("Add core device without filesystem."):
core_part.wipe_filesystem()
core = cache.add_core(core_part)
with TestRun.step("Fill exported object with data."):
dd = (Dd().input("/dev/zero").output(core.path).block_size(
Size(1, Unit.Blocks4096)).oflag("direct"))
dd.run()
with TestRun.step("Count dirty blocks on exported object."):
dirty_blocks_before = core.get_dirty_blocks()
with TestRun.step("Stop cache with option '--no-data-flush'."):
cache.stop(no_data_flush=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(
f"Expected caches count: 0; Actual caches count: {caches_count}."
)
with TestRun.step("Try to start cache without loading metadata."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_part.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
cli_messages.check_stderr_msg(
output, cli_messages.start_cache_with_existing_metadata)
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1 Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step(
"Compare dirty blocks number before and after loading cache."):
if dirty_blocks_before != core.get_dirty_blocks():
TestRun.LOGGER.error(
"Dirty blocks number is different than before loading cache.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_flush_inactive_devices():
"""
title: Negative test for flushing inactive CAS devices.
description: Validate that CAS prevents flushing dirty data from inactive CAS devices.
pass_criteria:
- No kernel error
- Exported object appears after plugging core device
- Flushing inactive CAS devices is possible neither by cleaning thread,
nor by calling cleaning methods
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core1", 1), ("core2", 1)])
cache_dev = devices["cache"].partitions[0]
first_core_dev = devices["core1"].partitions[0]
second_core_dev = devices["core2"].partitions[0]
plug_device = devices["core1"]
with TestRun.step("Start cache in WB mode and set alru cleaning policy."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cache.set_cleaning_policy(CleaningPolicy.alru)
cache.set_params_alru(
FlushParametersAlru(
staleness_time=timedelta(seconds=10),
wake_up_time=timedelta(seconds=1),
activity_threshold=timedelta(milliseconds=500)))
with TestRun.step("Add two cores."):
first_core = cache.add_core(first_core_dev)
second_core = cache.add_core(second_core_dev)
with TestRun.step(
"Create init config file using running CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run random writes to CAS device."):
run_fio([first_core.system_path, second_core.system_path])
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug one core disk."):
plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Wait longer than required for alru cleaning thread to start and verify "
"that dirty data is flushed only from active device."):
dirty_lines_before = {
first_core: first_core.get_dirty_blocks(),
second_core: second_core.get_dirty_blocks()
}
time.sleep(30)
check_amount_of_dirty_data(dirty_lines_before)
with TestRun.step("Try to call 'flush cache' command."):
dirty_lines_before = {
first_core: first_core.get_dirty_blocks(),
second_core: second_core.get_dirty_blocks()
}
try:
cache.flush_cache()
TestRun.fail(
"Flush cache operation should be blocked due to inactive cache devices, "
"but it executed successfully.")
except Exception as e:
TestRun.LOGGER.info(
f"Flush cache operation is blocked as expected.\n{str(e)}")
check_amount_of_dirty_data(dirty_lines_before)
with TestRun.step("Try to call 'flush core' command for inactive core."):
dirty_lines_before = {
first_core: first_core.get_dirty_blocks(),
second_core: second_core.get_dirty_blocks()
}
try:
first_core.flush_core()
TestRun.fail(
"Flush core operation should be blocked for inactive CAS devices, "
"but it executed successfully.")
except Exception as e:
TestRun.LOGGER.info(
f"Flush core operation is blocked as expected.\n{str(e)}")
check_amount_of_dirty_data(dirty_lines_before)
with TestRun.step(
"Plug core disk and verify that this change is reflected on the cache list."
):
plug_device.plug()
first_core.wait_for_status_change(CoreStatus.active)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache did not change status to 'running' after plugging core device. "
f"Actual state: {cache_status}.")
with TestRun.step("Stop cache."):
cache.stop()
def test_cache_stop_and_load(cache_mode):
"""
title: Test for stopping and loading cache back with dynamic cache mode switching.
description: |
Validate the ability of the CAS to switch cache modes at runtime and
check if all of them are working properly after switching and
after stopping and reloading cache back.
Check also other parameters consistency after reload.
pass_criteria:
- In all cache modes data reads and writes are handled properly before and after reload.
- All cache parameters preserve their values after reload.
"""
with TestRun.step("Partition cache and core devices"):
cache_dev, core_dev = storage_prepare()
with TestRun.step(f"Start cache in {cache_mode[0]} mode"):
cache = casadm.start_cache(cache_dev, cache_mode[0], force=True)
Udev.disable()
with TestRun.step("Add core to the cache"):
core = cache.add_core(core_dev)
with TestRun.step(f"Change cache mode to {cache_mode[1]}"):
cache.set_cache_mode(cache_mode[1], flush=True)
check_cache_config = cache.get_cache_config()
with TestRun.step(f"Check if {cache_mode[1]} cache mode works properly"):
check_cache_mode_operation(cache, core, cache_mode[1])
with TestRun.step("Stop and load cache back"):
cache.stop()
cache = casadm.load_cache(cache_dev)
with TestRun.step("Check parameters consistency"):
if check_cache_config != cache.get_cache_config():
failed_params = ""
if check_cache_config.cache_mode != cache.get_cache_mode():
failed_params += (
f"Cache mode is: {check_cache_config.cache_mode}, "
f"should be: {cache.get_cache_mode()}\n")
if check_cache_config.cleaning_policy != cache.get_cleaning_policy(
):
failed_params += (
f"Cleaning policy is: {check_cache_config.cleaning_policy}, "
f"should be: {cache.get_cleaning_policy()}\n")
if check_cache_config.cache_line_size != cache.get_cache_line_size(
):
failed_params += (
f"Cache line size is: {check_cache_config.cache_line_size}, "
f"should be: {cache.get_cache_line_size()}\n")
TestRun.fail(
f"Parameters do not match after reload:\n{failed_params}")
with TestRun.step(
f"Check if {cache_mode[1]} cache mode works properly after reload"
):
if cache_mode[1] == CacheMode.WA or cache_mode[1] == CacheMode.WO:
check_separated_read_write_after_reload(cache, core, cache_mode[1],
io_size)
else:
check_cache_mode_operation(cache, core, cache_mode[1])
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
Udev.enable()
def cache_load(cache_dev):
caches = []
for i in range(caches_count):
caches.append(casadm.load_cache(cache_dev.partitions[i]))
return caches
def test_recovery_unplug_cache_fs(cache_mode, cls, filesystem, direct):
"""
title: Test for recovery after cache drive removal - test with filesystem.
description: |
Verify that unflushed data can be safely recovered after, when SSD drive is removed
after write completion - test with filesystem.
pass_criteria:
- CAS recovers successfully after cache drive unplug
- No data corruption
"""
with TestRun.step("Prepare devices"):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(16, Unit.GibiByte)])
cache_device = cache_disk.partitions[0]
core_device = core_disk.partitions[0]
with TestRun.step("Create test files."):
source_file, target_file = create_test_files(test_file_size)
source_file_md5 = source_file.md5sum()
with TestRun.step("Create filesystem on core device."):
core_device.create_filesystem(filesystem)
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_device, cache_mode, cls)
core = cache.add_core(core_device)
with TestRun.step("Mount CAS device."):
core.mount(mount_point)
with TestRun.step("Copy file to CAS."):
copy_file(source=source_file.full_path, target=test_file_path,
size=test_file_size, direct="oflag" if direct else None)
TestRun.LOGGER.info(str(core.get_statistics()))
with TestRun.step("Unmount CAS device."):
core.unmount()
with TestRun.step("Unplug cache device."):
cache_disk.unplug()
TestRun.LOGGER.info(f"List caches:\n{casadm.list_caches().stdout}")
TestRun.LOGGER.info(f"Dirty blocks on cache: "
f"{cache.get_dirty_blocks().get_value(Unit.Blocks4096)}")
with TestRun.step("Stop cache."):
try:
cache.stop()
TestRun.fail("Stopping the cache should be aborted without --no-flush flag.")
except CmdException as e:
TestRun.LOGGER.info(str(e.output))
try:
cache.stop(no_data_flush=True)
TestRun.LOGGER.warning("Expected stopping cache with errors with --no-flush flag.")
except CmdException as e1:
cli_messages.check_stderr_msg(e1.output, cli_messages.stop_cache_errors)
with TestRun.step("Plug missing cache device."):
TestRun.LOGGER.info(str(casadm.list_caches(by_id_path=False)))
cache_disk.plug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_device)
TestRun.LOGGER.info(f"Dirty blocks on cache: "
f"{cache.get_dirty_blocks().get_value(Unit.Blocks4096)}")
with TestRun.step("Stop cache with data flush."):
cache.stop()
with TestRun.step("Mount core device."):
core_device.mount(mount_point)
with TestRun.step("Copy file from core device and check md5sum."):
copy_file(source=test_file_path, target=target_file.full_path,
size=test_file_size, direct="iflag" if direct else None)
target_file_md5 = target_file.md5sum()
compare_files(source_file_md5, target_file_md5)
with TestRun.step("Unmount core device and remove files."):
core_device.unmount()
try:
target_file.remove()
source_file.remove()
except Exception:
# On some OSes files at /tmp location are automatically removed after DUT hard reset
pass
def test_fault_power_hit(cache_mode):
"""
title: Test with power hit.
description: |
Test if there will be no metadata initialization after wake up
- when starting cache without initialization.
pass_criteria:
- Start cache without re-initialization failed.
- Start cache with load works correctly.
- Expected message found in log.
"""
with TestRun.step("Prepare CAS device."):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(2, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_dev = core_disk.partitions[0]
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
core = cache.add_core(core_dev)
with TestRun.step("Mark log lines for later validation of new entries."):
last_read_line = 1
log_lines = TestRun.executor.run_expect_success(
f"tail -qn +{last_read_line} {log_path}").stdout.splitlines()
last_read_line += len(log_lines)
with TestRun.step("Hard reset."):
power_control = TestRun.plugin_manager.get_plugin('power_control')
power_control.power_cycle()
with TestRun.step("Start cache without re-initialization."):
output = TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev=str(cache_dev.path),
cache_mode=str(cache_mode.name.lower()),
force=False,
load=False))
if cli_messages.check_stderr_msg(output, cli_messages.error_inserting_cache) and \
cli_messages.check_stderr_msg(output,
cli_messages.reinitialize_with_force_or_recovery):
TestRun.LOGGER.info(
f"Found expected exception: {cli_messages.error_inserting_cache}"
f" and {cli_messages.reinitialize_with_force_or_recovery}")
with TestRun.step("Start cache with load."):
try:
cache = casadm.load_cache(cache_dev)
TestRun.LOGGER.info(
f"Cache device loaded correctly (as expected).")
except CmdException as e:
TestRun.LOGGER.fail(
f"Failed to load cache device. Exception: {e.output}")
time.sleep(wait_short_time)
message_found = check_log(
last_read_line, cli_messages.reinitialize_with_force_or_recovery)
# check log second time in case that operation logging would take some more time
if not message_found:
time.sleep(wait_long_time)
result = check_log(
last_read_line,
cli_messages.reinitialize_with_force_or_recovery)
if not result:
TestRun.LOGGER.fail(
f"Haven't found expected message in the log.")
