def test_attach_core_to_incomplete_cache_volume():
"""
title: Test for attaching device to inactive cache volume.
description: |
Try to attach core device to inactive cache volume and check if it is visible in OS
properly.
pass_criteria:
- No kernel error
- Core status changes properly
- Cache loads with inactive core device
- Cache status changes properly
- Exported object is present only for active core
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 1)])
cache_dev = devices["cache"].partitions[0]
core_dev = devices["core"].partitions[0]
plug_device = devices["core"]
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, force=True)
core = cache.add_core(core_dev)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
with TestRun.step(
"Check if there is CAS device in /dev and core is in active status."
):
core.check_if_is_present_in_os()
core_status = core.get_status()
if core_status != CoreStatus.active:
TestRun.fail(
f"Core should be in active state. (Actual: {core_status})")
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Unplug core device."):
plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Check if there is no CAS device in /dev and core is in inactive status."
):
core.check_if_is_present_in_os(False)
if core.get_status() != CoreStatus.inactive:
TestRun.fail("Core should be in inactive state.")
with TestRun.step("Plug core device."):
plug_device.plug()
with TestRun.step(
"Check if core status changed to active and CAS device is visible in OS."
):
core.wait_for_status_change(CoreStatus.active)
core.check_if_is_present_in_os()
if cache.get_status() != CacheStatus.running:
TestRun.fail(
"Cache did not change status to 'running' after plugging core device."
)
def test_remove_detached_cores():
"""
title: Validate removing core devices from core pool.
description: Validate that it is possible to remove core devices from core pool.
pass_criteria:
- No kernel error
- All core devices are correctly added after plugging core disk.
- All cores are successfully removed.
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 4)])
cache_dev = devices["cache"].partitions[0]
core_devs = devices["core"].partitions
plug_device = devices["core"]
with TestRun.step("Start cache and add four cores."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cores = []
for d in core_devs:
cores.append(cache.add_core(d))
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run random writes to all CAS devices."):
run_fio([c.system_path for c in cores])
with TestRun.step(
"Flush dirty data from two CAS devices and verify than other two contain "
"dirty data."):
for core in cores:
if core.core_id % 2 == 0:
core.flush_core()
if core.get_dirty_blocks() != Size.zero():
TestRun.fail("Failed to flush CAS device.")
elif core.get_dirty_blocks() == Size.zero():
TestRun.fail("There should be dirty data on CAS device.")
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug core device from system and plug it back."):
plug_device.unplug()
time.sleep(2)
plug_device.plug()
with TestRun.step(
"Verify that all cores from plugged core device are listed with "
"proper status."):
for core in cores:
if core.get_status() != CoreStatus.detached:
TestRun.fail(f"Each core should be in detached state. "
f"Actual states: {casadm.list_caches().stdout}")
with TestRun.step("Remove CAS devices from core pool."):
casadm.remove_all_detached_cores()
with TestRun.step("Verify that cores are no longer listed."):
output = casadm.list_caches().stdout
for dev in core_devs:
if dev.system_path in output:
TestRun.fail(
f"CAS device is still listed in casadm list output:\n{output}"
)
def test_stress_service(cache_mode):
"""
title: Stress test for starting and stopping CAS service.
description: |
Validate the ability of CAS to restart CAS service
and load CAS device in the loop.
pass_criteria:
- No system crash while restarting CAS service or loading cache.
- CAS service restarts with no errors.
- CAS device loads successfully.
"""
with TestRun.step("Prepare cache and core."):
cache_dev, core_dev = prepare()
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
casadm.add_core(cache, core_dev)
for _ in TestRun.iteration(
range(0, iterations_per_config),
f"Stop and start CAS service {iterations_per_config} times."):
with TestRun.step(
"Create CAS init config based on current running CAS configuration."
):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop CAS service."):
casctl.stop()
with TestRun.step("Check if service stopped successfully."):
if len(casadm_parser.get_caches()) != 0:
TestRun.fail(
"There are still running caches after stopping service.")
if len(casadm_parser.get_cores(cache.cache_id)) != 0:
TestRun.fail(
"There are still running cores after stopping service.")
with TestRun.step("Start CAS service."):
casctl.start()
time.sleep(1) # Time for CAS devices to start
with TestRun.step("Check if CAS configuration loaded successfully."):
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}."
)
with TestRun.step("Stop caches and create default init config file."):
casadm.stop_all_caches()
InitConfig.create_default_init_config()
def test_stress_dirty_shutdown(cache_line_size, cache_mode, cleaning_policy):
"""
title: Stress test for dirty shutdowns during IO workload.
description: |
Validate the ability of CAS to start cache instances upon system boot after
dirty shutdown during IO workloads.
pass_criteria:
- No system crash.
- CAS loads correctly after DUT hard reset.
"""
with TestRun.step("Prepare devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(5, Unit.GibiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_sizes = [Size(5, Unit.GibiByte)] * cores_number
core_dev.create_partitions(core_sizes)
with TestRun.step(
"Start cache according to configuration and add core devices."):
cache = casadm.start_cache(cache_part,
cache_mode,
cache_line_size,
force=True)
if cleaning_policy is not None:
cache.set_cleaning_policy(cleaning_policy)
exported_objects = []
for i in range(0, cores_number):
exported_objects.append(cache.add_core(core_dev.partitions[i]))
with TestRun.step(
"Create CAS init configuration file based on running configuration."
):
InitConfig.create_init_config_from_running_configuration()
for _ in TestRun.iteration(range(0, iterations_per_config),
"Load cache after reboot while heavy IO."):
with TestRun.step("Start heavy IO workload on both CAS devices."):
run_io(exported_objects)
time.sleep(120)
with TestRun.step("Reset platform."):
power_control = TestRun.plugin_manager.get_plugin('power_control')
power_control.power_cycle()
with TestRun.step("Check configuration after load."):
check_configuration(cleaning_policy, cache_mode, cache_line_size)
with TestRun.step("Stop cache."):
cache.stop()
def test_udev_core():
"""
title: CAS udev rule execution for core after detaching and re-attaching cache device.
description: |
Verify if CAS udev rule places core in the core pool when cache instance assigned to this
core is not available in system and attaches core to cache
when cache is plugged and successfully loaded.
pass_criteria:
- No kernel error
- Core devices are listed in core pool when cache is not available
- Core devices are moved from core pool and attached to cache after plugging cache device
"""
with TestRun.step("Start cache and add core."):
cache_disk = TestRun.disks["cache"]
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks["core"]
core_disk.create_partitions([Size(2, Unit.GibiByte)])
core_dev = core_disk.partitions[0]
cache = casadm.start_cache(cache_dev, force=True)
core = cache.add_core(core_dev)
with TestRun.step("Create init config from running CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Unplug core disk."):
core_disk.unplug()
with TestRun.step("Plug core disk."):
core_disk.plug()
time.sleep(1)
with TestRun.step("Check if core device is listed in core pool."):
check_if_dev_in_core_pool(core_dev)
with TestRun.step("Unplug cache disk."):
cache_disk.unplug()
with TestRun.step("Plug cache disk."):
cache_disk.plug()
with TestRun.step(
"Check if core device is active and not in the core pool."):
check_if_dev_in_core_pool(core_dev, False)
if core.get_status() != CoreStatus.active:
TestRun.fail(
f"Core status is {core.get_status()} instead of active.")
def test_udev_cache_load(cache_mode):
"""
title: CAS udev rule execution after unplugging and plugging cache device.
description: |
Verify if CAS udev rule is executed after unplugging and plugging cache device and if
cache is properly loaded.
pass_criteria:
- No kernel error
- Cache is properly loaded after plugging cache device
- Cache is not loaded after unplugging cache device
"""
with TestRun.step("Start cache."):
cache_disk = TestRun.disks["cache"]
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
cache = casadm.start_cache(cache_dev, cache_mode=cache_mode)
with TestRun.step("Create init config from running configuration"):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Unplug cache disk."):
cache_disk.unplug()
with TestRun.step("Plug cache disk."):
cache_disk.plug()
time.sleep(1)
with TestRun.step("List caches and check if cache is loaded."):
caches = casadm_parser.get_caches()
if len(caches) < 1:
TestRun.fail("Cache did not load.")
elif len(caches) > 1:
caches_list = '\n'.join(caches)
TestRun.fail(f"There is more than 1 cache loaded:\n{caches_list}")
elif caches[0].cache_device.path != cache_dev.path:
TestRun.fail(f"Cache loaded on wrong device. "
f"Actual: {caches[0].cache_device.path}, "
f"expected: {cache_dev.path}")
elif caches[0].get_cache_mode() != cache_mode:
TestRun.fail(
f"Cache did load with different cache mode. "
f"Actual: {caches[0].get_cache_mode()}, expected: {cache_mode}"
)
TestRun.LOGGER.info("Cache is correctly loaded.")
def test_udev_core_partition():
"""
title: |
CAS udev rule execution after re-attaching partitions existing in configuration file as
cores.
description: |
Verify if CAS udev rule is executed when partitions existing in CAS configuration file
as cores are being attached.
pass_criteria:
- No kernel error
- Created partitions are added to core pool after attaching core drive.
"""
cores_count = 4
with TestRun.step(
"Create four partitions on core device and one on cache device."):
cache_disk = TestRun.disks["cache"]
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks["core"]
core_disk.create_partitions([Size(2, Unit.GibiByte)] * cores_count)
core_devices = core_disk.partitions
with TestRun.step("Start cache and add created partitions as cores."):
cache = casadm.start_cache(cache_dev, force=True)
for dev in core_devices:
cache.add_core(dev)
with TestRun.step("Create init config from running CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Detach core disk."):
core_disk.unplug()
with TestRun.step("Plug missing core disk."):
core_disk.plug()
time.sleep(1)
with TestRun.step(
"List cache devices and check that created partitions are present "
"in core pool."):
for dev in core_devices:
check_if_dev_in_core_pool(dev)
def test_load_cache_with_inactive_core():
"""
title: Load cache with unavailable core devices.
description: Check if it is possible to load cache with unavailable core devices.
pass_criteria:
- No kernel error
- It is possible to perform cache load operation with unavailable devices.
- Warning message about not available core device should appear.
- Cache status should change to active after plugging missing core device.
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 1)])
cache_dev = devices["cache"].partitions[0]
core_dev = devices["core"].partitions[0]
plug_device = devices["core"]
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, force=True)
core = cache.add_core(core_dev)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Unplug core device."):
plug_device.unplug()
with TestRun.step("Load cache."):
output = TestRun.executor.run(cli.load_cmd(cache_dev.path))
cli_messages.check_stderr_msg(output,
cli_messages.load_inactive_core_missing)
with TestRun.step("Plug missing device and stop cache."):
plug_device.plug()
time.sleep(1)
core.wait_for_status_change(CoreStatus.active)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache did not change status to 'running' after plugging core device. "
f"Actual state: {cache_status}.")
cache.stop()
def test_udev_raid_core():
"""
title: CAS udev rule execution for core after recreating RAID device existing in
configuration file as core.
description: |
Verify if CAS udev rule is executed for RAID volume recreated after soft reboot.
pass_criteria:
- No kernel error
- After reboot, the RAID volume is added to the cache instance and is in 'active' state
"""
with TestRun.step("Test prepare."):
cache_disk = TestRun.disks["cache"]
cache_disk.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks["core"]
core_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk = core_disk.partitions[0]
core_disk2 = TestRun.disks["core2"]
core_disk2.create_partitions([Size(2, Unit.GibiByte)])
core_disk2 = core_disk2.partitions[0]
with TestRun.step("Create RAID0 volume."):
config = RaidConfiguration(level=Level.Raid0,
metadata=MetadataVariant.Legacy,
number_of_devices=2)
core_dev = Raid.create(config, [core_disk, core_disk2])
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, force=True)
core = cache.add_core(core_dev)
with TestRun.step("Create init config from running CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Reboot system."):
TestRun.executor.reboot()
with TestRun.step(
"Check if core device is active and not in the core pool."):
check_if_dev_in_core_pool(core_dev, False)
if core.get_status() != CoreStatus.active:
TestRun.fail(
f"Core status is {core.get_status()} instead of active.")
def test_remove_inactive_devices():
"""
title: Validate removing inactive CAS devices.
description: |
Validate that it is possible to remove inactive CAS devices when there are no dirty
cache lines associated with them and that removing CAS devices is prevented otherwise
(unless ‘force’ option is used).
pass_criteria:
- No kernel error
- Removing CAS devices without dirty data is successful.
- Removing CAS devices with dirty data without ‘force’ option is blocked.
- Removing CAS devices with dirty data with ‘force’ option is successful.
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 4)])
cache_dev = devices["cache"].partitions[0]
core_devs = devices["core"].partitions
plug_device = devices["core"]
with TestRun.step("Start cache and add four cores."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cores = []
for d in core_devs:
cores.append(cache.add_core(d))
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run random writes to all CAS devices."):
run_fio([c.system_path for c in cores])
with TestRun.step(
"Flush dirty data from two CAS devices and verify than other two "
"contain dirty data."):
for core in cores:
if core.core_id % 2 == 0:
core.flush_core()
if core.get_dirty_blocks() != Size.zero():
TestRun.fail("Failed to flush CAS device.")
elif core.get_dirty_blocks() == Size.zero():
TestRun.fail("There should be dirty data on CAS device.")
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug core disk."):
plug_device.unplug()
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
with TestRun.step(
"Verify that all previously created CAS devices are listed with "
"proper status."):
for core in cores:
if core.get_status() != CoreStatus.inactive:
TestRun.fail(f"Each core should be in inactive state. "
f"Actual states:\n{casadm.list_caches().stdout}")
with TestRun.step(
"Try removing CAS device without ‘force’ option. Verify that for "
"dirty CAS devices operation is blocked, proper message is displayed "
"and device is still listed."):
shuffle(cores)
for core in cores:
try:
dirty_blocks = core.get_dirty_blocks()
core.remove_core()
if dirty_blocks != Size.zero():
TestRun.fail(
"Removing dirty CAS device should be impossible but remove "
"command executed without any error.")
TestRun.LOGGER.info(
"Removing core with force option skipped for clean CAS device."
)
except CmdException as e:
if dirty_blocks == Size.zero():
TestRun.fail(
"Removing clean CAS device should be possible but remove "
"command returned an error.")
TestRun.LOGGER.info(
"Remove operation without force option is blocked for "
"dirty CAS device as expected.")
cli_messages.check_stderr_msg(
e.output, cli_messages.remove_inactive_core)
output = casadm.list_caches().stdout
if core.system_path not in output:
TestRun.fail(
f"CAS device is not listed in casadm list output but it should be."
f"\n{output}")
core.remove_core(force=True)
with TestRun.step("Plug missing disk and stop cache."):
plug_device.plug()
casadm.stop_all_caches()
def test_cas_startup(cache_mode, filesystem):
"""
title: Test for starting CAS on system startup.
pass_criteria:
- System does not crash.
- CAS modules are loaded before partitions are mounted.
- Cache is loaded before partitions are mounted.
- Exported object is mounted after startup is complete.
"""
with TestRun.step(
"Prepare partitions for cache (200MiB) and for core (400MiB)"):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(200, Unit.MebiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(400, Unit.MebiByte)])
core_part = core_dev.partitions[0]
with TestRun.step("Start cache and add core"):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
core = cache.add_core(core_part)
with TestRun.step("Create and mount filesystem"):
core.create_filesystem(filesystem)
core.mount(mountpoint)
with TestRun.step("Create test file and calculate md5 checksum"):
(Dd().input("/dev/urandom").output(filepath).count(16).block_size(
Size(1, Unit.MebiByte)).run())
test_file = File(filepath)
md5_before = test_file.md5sum()
with TestRun.step("Add mountpoint fstab and create intelcas.conf"):
fstab.add_mountpoint(device=core,
mount_point=mountpoint,
fs_type=filesystem)
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Reboot"):
TestRun.executor.reboot()
with TestRun.step("Check if cache is started"):
caches = list(get_caches())
if len(caches) != 1:
TestRun.fail(f"Expected one cache, got {len(caches)}!")
if caches[0].cache_id != cache.cache_id:
TestRun.fail("Invalid cache id!")
with TestRun.step("Check if core is added"):
cores = list(get_cores(cache.cache_id))
if len(cores) != 1:
TestRun.fail(f"Expected one core, got {len(cores)}!")
if cores[0].core_id != core.core_id:
TestRun.fail("Invalid core id!")
with TestRun.step("Check if filesystem is mounted"):
if not core.is_mounted():
TestRun.fail("Core is not mounted!")
with TestRun.step("Check if md5 checksum matches"):
md5_after = test_file.md5sum()
if md5_before != md5_after:
TestRun.fail("md5 checksum mismatch!")
with TestRun.step("Test cleanup"):
fstab.remove_mountpoint(device=core)
core.unmount()
InitConfig.create_default_init_config()
casadm.stop_all_caches()
def test_user_service():
"""
title: Test that OpenCAS does not allow to change service status by non-root user.
description: |
Verify that changing OpenCAS service status by non-root user is forbidden by OpenCAS.
pass_criteria:
- Non-root user cannot change OpenCAS service state.
- Non-root sudoer user can change OpenCAS service state.
"""
with TestRun.step("Prepare cache and core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)])
cache_dev = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(2, Unit.GibiByte)])
core_dev = core_dev.partitions[0]
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, force=True)
with TestRun.step("Add core to cache and mount it."):
core_dev.create_filesystem(Filesystem.ext3)
core = cache.add_core(core_dev)
core.mount(mount_point)
with TestRun.step("Create 'opencas.conf' from running configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step(f"Copy casadm bin from {system_casadm_bin_path} "
f"to {user_casadm_bin_dest_path}."):
casadm_bin = fs_utils.parse_ls_output(fs_utils.ls_item(f"{system_casadm_bin_path}"))[0]
casadm_bin_copy = casadm_bin.copy(user_casadm_bin_dest_path, True)
casadm_bin_copy.chmod_numerical(777)
with TestRun.step("Unmount core."):
core.unmount()
with TestRun.step("Add non-root user account."):
TestRun.executor.run(f"useradd -N -r -l {user_name}")
user_home_dir = fs_utils.parse_ls_output(fs_utils.ls_item(f"/home/{user_name}"))[0]
user_home_dir.chmod_numerical(777, True)
with TestRun.step("Try to stop OpenCAS service."):
try:
output = run_as_other_user(cli.ctl_stop(False), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Stopping OpenCAS should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot stop OpenCAS.")
with TestRun.step("Try to start OpenCAS service."):
try:
output = run_as_other_user(cli.ctl_start(), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Starting OpenCAS should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot start OpenCAS.")
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
with TestRun.step("Try to init OpenCAS service."):
try:
output = run_as_other_user(cli.ctl_init(True), user_name)
if output.exit_code == 0:
TestRun.LOGGER.error("Initiating OpenCAS should fail!")
except CmdException:
TestRun.LOGGER.info("Non-root user cannot init OpenCAS.")
with TestRun.step("Add non-root user account to sudoers group."):
TestRun.executor.run(f'echo "{user_name} ALL = (root) NOPASSWD:ALL" '
f'| sudo tee /etc/sudoers.d/{user_name}')
with TestRun.step("Try to stop OpenCAS service with 'sudo'."):
try:
run_as_other_user(cli.ctl_stop(False), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to stop OpenCAS.")
with TestRun.step("Try to start OpenCAS service with 'sudo'."):
try:
run_as_other_user(cli.ctl_start(), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to start OpenCAS.")
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
with TestRun.step("Try to init OpenCAS service with 'sudo'."):
try:
run_as_other_user(cli.ctl_init(True), user_name, True)
except CmdException:
TestRun.LOGGER.error("Non-root sudoer user should be able to init OpenCAS.")
with TestRun.step("Remove user account."):
TestRun.executor.run(f"userdel -r -Z {user_name}")
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_init_reboot_runlevels(runlevel, cache_mode):
"""
title: Initialize CAS devices after reboot
description: |
Verify that CAS init script starts cache properly after reboot in different runlevels.
pass_criteria:
- Cache should be loaded successfully after reboot.
"""
with TestRun.step(f"Set runlevel to {runlevel.value}."):
os_utils.change_runlevel(runlevel)
with TestRun.step("Prepare CAS device."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
cache_dev = cache_disk.partitions[0]
core_disk = TestRun.disks['core']
core_disk.create_partitions([Size(1, Unit.GibiByte)])
core_dev = core_disk.partitions[0]
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
core = cache.add_core(core_dev)
with TestRun.step("Create CAS init config based on running configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Make filesystem on CAS device and mount it."):
core.create_filesystem(Filesystem.xfs)
core.mount(mount_point)
with TestRun.step("Start writing file to CAS."):
fio = Fio().create_command()\
.file_name(os.path.join(mount_point, "test_file"))\
.read_write(ReadWrite.randwrite)\
.io_engine(IoEngine.sync)\
.num_jobs(1).direct()\
.file_size(Size(30, Unit.GibiByte))
fio.run_in_background()
os_utils.sync()
os_utils.drop_caches()
time.sleep(10)
TestRun.executor.run_expect_success("pgrep fio")
with TestRun.step("Reboot machine during writing a file."):
TestRun.executor.reboot()
with TestRun.step("Check if cache was properly started at boot time"):
# Wait for CAS to load after boot
time.sleep(60)
caches = casadm_parser.get_caches()
if len(caches) == 1:
TestRun.LOGGER.info("Cache started properly at boot time.")
else:
TestRun.LOGGER.error("Cache did not start properly at boot time.")
with TestRun.step("Stop cache and set default runlevel."):
if len(caches) != 0:
casadm.stop_all_caches()
os_utils.change_runlevel(Runlevel.runlevel3)
TestRun.executor.reboot()
def test_flush_inactive_devices():
"""
title: Negative test for flushing inactive CAS devices.
description: Validate that CAS prevents flushing dirty data from inactive CAS devices.
pass_criteria:
- No kernel error
- Exported object appears after plugging core device
- Flushing inactive CAS devices is possible neither by cleaning thread,
nor by calling cleaning methods
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core1", 1), ("core2", 1)])
cache_dev = devices["cache"].partitions[0]
first_core_dev = devices["core1"].partitions[0]
second_core_dev = devices["core2"].partitions[0]
plug_device = devices["core1"]
with TestRun.step("Start cache in WB mode and set alru cleaning policy."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cache.set_cleaning_policy(CleaningPolicy.alru)
cache.set_params_alru(
FlushParametersAlru(
staleness_time=timedelta(seconds=10),
wake_up_time=timedelta(seconds=1),
activity_threshold=timedelta(milliseconds=500)))
with TestRun.step("Add two cores."):
first_core = cache.add_core(first_core_dev)
second_core = cache.add_core(second_core_dev)
with TestRun.step(
"Create init config file using running CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run random writes to CAS device."):
run_fio([first_core.system_path, second_core.system_path])
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug one core disk."):
plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Wait longer than required for alru cleaning thread to start and verify "
"that dirty data is flushed only from active device."):
dirty_lines_before = {
first_core: first_core.get_dirty_blocks(),
second_core: second_core.get_dirty_blocks()
}
time.sleep(30)
check_amount_of_dirty_data(dirty_lines_before)
with TestRun.step("Try to call 'flush cache' command."):
dirty_lines_before = {
first_core: first_core.get_dirty_blocks(),
second_core: second_core.get_dirty_blocks()
}
try:
cache.flush_cache()
TestRun.fail(
"Flush cache operation should be blocked due to inactive cache devices, "
"but it executed successfully.")
except Exception as e:
TestRun.LOGGER.info(
f"Flush cache operation is blocked as expected.\n{str(e)}")
check_amount_of_dirty_data(dirty_lines_before)
with TestRun.step("Try to call 'flush core' command for inactive core."):
dirty_lines_before = {
first_core: first_core.get_dirty_blocks(),
second_core: second_core.get_dirty_blocks()
}
try:
first_core.flush_core()
TestRun.fail(
"Flush core operation should be blocked for inactive CAS devices, "
"but it executed successfully.")
except Exception as e:
TestRun.LOGGER.info(
f"Flush core operation is blocked as expected.\n{str(e)}")
check_amount_of_dirty_data(dirty_lines_before)
with TestRun.step(
"Plug core disk and verify that this change is reflected on the cache list."
):
plug_device.plug()
first_core.wait_for_status_change(CoreStatus.active)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache did not change status to 'running' after plugging core device. "
f"Actual state: {cache_status}.")
with TestRun.step("Stop cache."):
cache.stop()
def test_print_statistics_inactive(cache_mode):
"""
title: Print statistics for cache with inactive cache volumes.
description: |
Check if statistics are displayed properly when there is one or more
inactive cache volumes added to cache.
pass_criteria:
- No kernel error
- All statistics should contain appropriate information depending on situation of
cache and core devices (as described in test steps)
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core1", 1), ("core2", 1)])
cache_dev = devices["cache"].partitions[0]
first_core_dev = devices["core1"].partitions[0]
second_core_dev = devices["core2"].partitions[0]
first_plug_device = devices["core1"]
second_plug_device = devices["core2"]
Udev.disable(
) # disabling udev for a while prevents creating clean data on cores
with TestRun.step("Start cache and add cores."):
cache = casadm.start_cache(cache_dev,
cache_mode=cache_mode,
force=True)
first_core = cache.add_core(first_core_dev)
second_core = cache.add_core(second_core_dev)
cache_mode_traits = CacheMode.get_traits(cache.get_cache_mode())
with TestRun.step("Disable cleaning and sequential cutoff policies."):
cache.set_cleaning_policy(CleaningPolicy.nop)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run IO."):
run_fio([first_core.path, second_core.path])
with TestRun.step(
"Print statistics and check if there is no inactive usage section."
):
active_stats = cache.get_statistics()
check_if_inactive_section_exists(active_stats, False)
with TestRun.step("Stop cache."):
if CacheModeTrait.LazyWrites in cache_mode_traits:
cache.stop(no_data_flush=True)
else:
cache.stop()
with TestRun.step("Remove both core devices from OS."):
Udev.enable() # enable udev back because it's necessary now
first_plug_device.unplug()
second_plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Check if inactive devices section appeared and contains appropriate "
"information."):
inactive_stats_before = cache.get_statistics()
check_if_inactive_section_exists(inactive_stats_before)
check_number_of_inactive_devices(inactive_stats_before, 2)
with TestRun.step(
"Attach one of detached core devices and add it to cache."):
first_plug_device.plug()
time.sleep(1)
first_core_status = first_core.get_status()
if first_core_status != CoreStatus.active:
TestRun.fail(
f"Core {first_core.path} should be in active state but it is not. "
f"Actual state: {first_core_status}.")
with TestRun.step("Check cache statistics section of inactive devices."):
inactive_stats_after = cache.get_statistics()
check_if_inactive_section_exists(inactive_stats_after)
check_number_of_inactive_devices(inactive_stats_after, 1)
# criteria for checks below
insert_write_traits = CacheModeTrait.InsertWrite in cache_mode_traits
lazy_write_traits = CacheModeTrait.LazyWrites in cache_mode_traits
lazy_writes_or_no_insert_write_traits = (not insert_write_traits
or lazy_write_traits)
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_occupancy,
inactive_stats_after.inactive_usage_stats.inactive_occupancy,
"inactive occupancy", not insert_write_traits)
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_clean,
inactive_stats_after.inactive_usage_stats.inactive_clean,
"inactive clean", lazy_writes_or_no_insert_write_traits)
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_dirty,
inactive_stats_after.inactive_usage_stats.inactive_dirty,
"inactive dirty", not lazy_write_traits)
with TestRun.step("Check statistics per inactive core."):
inactive_core_stats = second_core.get_statistics()
if inactive_stats_after.inactive_usage_stats.inactive_occupancy == \
inactive_core_stats.usage_stats.occupancy:
TestRun.LOGGER.info(
"Inactive occupancy in cache statistics is equal to inactive core "
"occupancy.")
else:
TestRun.fail(
f"Inactive core occupancy ({inactive_core_stats.usage_stats.occupancy}) "
f"should be the same as cache inactive occupancy "
f"({inactive_stats_after.inactive_usage_stats.inactive_occupancy})."
)
with TestRun.step(
"Remove inactive core from cache and check if cache is in running state."
):
cache.remove_core(second_core.core_id, force=True)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache did not change status to 'running' after plugging core device. "
f"Actual status: {cache_status}.")
with TestRun.step(
"Check if there is no inactive devices statistics section and if cache has "
"Running status."):
cache_stats = cache.get_statistics()
check_if_inactive_section_exists(cache_stats, False)
check_number_of_inactive_devices(cache_stats, 0)
with TestRun.step("Plug missing disk and stop cache."):
second_plug_device.plug()
time.sleep(1)
cache.stop()
def test_list_cache_and_cache_volumes():
"""
title: List cache with cache volumes and check their status.
description: |
Check if casadm command correctly lists caches and cache volumes with their statuses.
pass_criteria:
- No kernel error
- Output of list command should be correct in each case (as described in test steps)
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 1)])
cache_dev = devices["cache"].partitions[0]
core_dev = devices["core"].partitions[0]
plug_device = devices["core"]
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, force=True)
core = cache.add_core(core_dev)
with TestRun.step(
"Check if list caches command shows proper output (cache should have status "
"Running and cache volume should be Active)."):
core_status = core.get_status()
if core_status != CoreStatus.active:
TestRun.fail(
f"Core should be in active state. Actual state: {core_status}."
)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache should be in running state. Actual state: {cache_status}"
)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Unplug core device."):
plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Check if list cache command shows proper output (cache should have status "
"Incomplete and cache volume should be Inactive)."):
core_status = core.get_status()
if core_status != CoreStatus.inactive:
TestRun.fail(
f"Core should be in inactive state. Actual state: {core_status}."
)
cache_status = cache.get_status()
if cache_status != CacheStatus.incomplete:
TestRun.fail(
f"Cache should be in incomplete state. Actual state: {cache_status}."
)
with TestRun.step("Plug missing device and stop cache."):
plug_device.plug()
core.wait_for_status_change(CoreStatus.active)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache did not change status to 'running' after plugging core device. "
f"Actual state: {cache_status}")
cache.stop()
def test_print_statistics_inactive(cache_mode):
"""
title: Print statistics for cache with inactive cache volumes.
description: |
Check if statistics are displayed properly when there is one or more
inactive cache volumes added to cache.
pass_criteria:
- No kernel error
- All statistics should contain appropriate information depending on situation of
cache and core devices (as described in test steps)
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core1", 1), ("core2", 1)])
cache_dev = devices["cache"].partitions[0]
first_core_dev = devices["core1"].partitions[0]
second_core_dev = devices["core2"].partitions[0]
first_plug_device = devices["core1"]
second_plug_device = devices["core2"]
with TestRun.step("Start cache and add cores."):
cache = casadm.start_cache(cache_dev,
cache_mode=cache_mode,
force=True)
first_core = cache.add_core(first_core_dev)
second_core = cache.add_core(second_core_dev)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run IO."):
run_fio([first_core.system_path, second_core.system_path])
with TestRun.step(
"Print statistics and check if there is no inactive usage section."
):
active_stats = cache.get_statistics()
check_if_inactive_section_exists(active_stats, False)
with TestRun.step("Stop cache."):
cache.stop()
with TestRun.step("Remove both core devices from OS."):
first_plug_device.unplug()
second_plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Check if inactive devices section appeared and contains appropriate "
"information."):
inactive_stats_before = cache.get_statistics()
check_if_inactive_section_exists(inactive_stats_before)
check_number_of_inactive_devices(inactive_stats_before, 2)
with TestRun.step(
"Attach one of detached core devices and add it to cache."):
first_plug_device.plug()
first_core_status = first_core.get_status()
if first_core_status != CoreStatus.active:
TestRun.fail(
f"Core {first_core.system_path} should be in active state but it is not. "
f"Actual state: {first_core_status}.")
with TestRun.step("Check cache statistics section of inactive devices."):
inactive_stats_after = cache.get_statistics()
check_if_inactive_section_exists(inactive_stats_after)
check_number_of_inactive_devices(inactive_stats_after, 1)
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_occupancy,
inactive_stats_after.inactive_usage_stats.inactive_occupancy,
"inactive occupancy",
cache.get_cache_mode() == CacheMode.PT)
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_clean,
inactive_stats_after.inactive_usage_stats.inactive_clean,
"inactive clean",
cache.get_cache_mode() in [CacheMode.PT, CacheMode.WB])
check_inactive_usage_stats(
inactive_stats_before.inactive_usage_stats.inactive_dirty,
inactive_stats_after.inactive_usage_stats.inactive_dirty,
"inactive dirty",
cache.get_cache_mode() != CacheMode.WB)
with TestRun.step("Check statistics per inactive core."):
inactive_core_stats = second_core.get_statistics()
if inactive_stats_after.inactive_usage_stats.inactive_occupancy == \
inactive_core_stats.usage_stats.occupancy:
TestRun.LOGGER.info(
"Inactive occupancy in cache statistics is equal to inactive core "
"occupancy.")
else:
TestRun.fail(
f"Inactive core occupancy ({inactive_core_stats.usage_stats.occupancy}) "
f"should be the same as cache inactive occupancy "
f"({inactive_stats_after.inactive_usage_stats.inactive_occupancy})."
)
with TestRun.step(
"Remove inactive core from cache and check if cache is in running state."
):
cache.remove_core(second_core.core_id, force=True)
cache_status = cache.get_status()
if cache_status != CacheStatus.running:
TestRun.fail(
f"Cache did not change status to 'running' after plugging core device. "
f"Actual status: {cache_status}.")
with TestRun.step(
"Check if there is no inactive devices statistics section and if cache has "
"Running status."):
cache_stats = cache.get_statistics()
check_if_inactive_section_exists(cache_stats, False)
check_number_of_inactive_devices(cache_stats, 0)
with TestRun.step("Plug missing disk and stop cache."):
second_plug_device.plug()
cache.stop()
def test_preserve_data_for_inactive_device():
"""
title: Validate preserving data for inactive CAS devices.
description: Validate that cached data for inactive CAS devices is preserved.
pass_criteria:
- No kernel error
- File md5 checksums match in every iteration.
- Cache read hits increase after reads (md5 checksum) from CAS device with attached core.
"""
mount_dir = "/mnt/test"
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 1)])
cache_dev = devices["cache"].partitions[0]
core_dev = devices["core"].partitions[0]
plug_device = devices["core"]
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
cache.set_cleaning_policy(CleaningPolicy.nop)
core = cache.add_core(core_dev)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Create filesystem on CAS device and mount it."):
core.create_filesystem(Filesystem.ext3)
core.mount(mount_dir)
with TestRun.step(
"Create a test file with random writes on mount point and count it's md5."
):
file_path = f"{mount_dir}/test_file"
test_file = File.create_file(file_path)
dd = Dd().input("/dev/random") \
.output(file_path) \
.count(100) \
.block_size(Size(1, Unit.Blocks512))
dd.run()
os_utils.sync()
md5_after_create = test_file.md5sum()
cache_stats_before_stop = cache.get_statistics()
core_stats_before_stop = core.get_statistics()
with TestRun.step("Unmount CAS device."):
core.unmount()
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug core device."):
plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
cache_stats_after_load = cache.get_statistics()
core_stats_after_load = core.get_statistics()
if cache_stats_before_stop.usage_stats.clean != cache_stats_after_load.usage_stats.clean or\
cache_stats_before_stop.usage_stats.dirty != \
cache_stats_after_load.usage_stats.dirty or\
core_stats_before_stop.usage_stats.clean != \
core_stats_after_load.usage_stats.clean or\
core_stats_before_stop.usage_stats.dirty != core_stats_after_load.usage_stats.dirty:
TestRun.fail(
f"Statistics after counting md5 are different than after cache load.\n"
f"Cache stats before: {cache_stats_before_stop}\n"
f"Cache stats after: {cache_stats_after_load}\n"
f"Core stats before: {core_stats_before_stop}\n"
f"Core stats after: {core_stats_after_load}")
with TestRun.step(
"Plug core disk using sysfs and verify this change is reflected "
"on the cache list."):
plug_device.plug()
if cache.get_status() != CacheStatus.running or core.get_status(
) != CoreStatus.active:
TestRun.fail(
f"Expected cache status is running (actual - {cache.get_status()}).\n"
f"Expected core status is active (actual - {core.get_status()})."
)
with TestRun.step("Mount CAS device"):
core.mount(mount_dir)
with TestRun.step(
"Count md5 checksum for test file and compare it with previous value."
):
cache_read_hits_before_md5 = cache.get_statistics(
).request_stats.read.hits
md5_after_cache_load = test_file.md5sum()
if md5_after_create != md5_after_cache_load:
TestRun.fail(
"Md5 checksum after cache load operation is different than before "
"stopping cache.")
else:
TestRun.LOGGER.info(
"Md5 checksum is identical before and after cache load operation "
"with inactive CAS device.")
with TestRun.step(
"Verify that cache read hits increased after counting md5 checksum."
):
cache_read_hits_after_md5 = cache.get_statistics(
).request_stats.read.hits
if cache_read_hits_after_md5 - cache_read_hits_before_md5 < 0:
TestRun.fail(
f"Cache read hits did not increase after counting md5 checksum. "
f"Before: {cache_read_hits_before_md5}. "
f"After: {cache_read_hits_after_md5}.")
else:
TestRun.LOGGER.info("Cache read hits increased as expected.")
with TestRun.step("Unmount CAS device and stop cache."):
core.unmount()
cache.stop()
def test_stop_cache_with_inactive_devices():
"""
title: Validate stopping cache with inactive CAS devices.
description: |
Validate that cache with inactive CAS devices cannot be stopped
unless ‘force’ option is used.
pass_criteria:
- No kernel error
- Stopping cache with inactive CAS devices without ‘force’ option is blocked.
- Stopping cache with inactive CAS devices with ‘force’ option is successful.
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 1)])
cache_dev = devices["cache"].partitions[0]
core_dev = devices["core"].partitions[0]
plug_device = devices["core"]
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
core = cache.add_core(core_dev)
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step(
"Run random writes and verify that CAS device contains dirty data."
):
run_fio([core.system_path])
if core.get_dirty_blocks() == Size.zero():
TestRun.fail("There is no dirty data on core device.")
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug core disk."):
plug_device.unplug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step(
"Verify that previously created CAS device is listed with proper status."
):
core_status = core.get_status()
if core_status != CoreStatus.inactive:
TestRun.fail(
f"CAS device should be in inactive state. Actual status: {core_status}."
)
with TestRun.step(
"Try stopping cache without ‘no data flush’ option, verify that operation "
"was blocked and proper message is displayed."):
try_stop_incomplete_cache(cache)
with TestRun.step("Stop cache with force option."):
cache.stop(no_data_flush=True)
with TestRun.step("Plug missing core device."):
plug_device.plug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_dev)
with TestRun.step("Stop cache with flushing dirty data."):
cache.stop()
with TestRun.step("Unplug core device."):
plug_device.unplug()
with TestRun.step("Load cache and verify core status is inactive."):
cache = casadm.load_cache(cache_dev)
core_status = core.get_status()
if core_status != CoreStatus.inactive:
TestRun.fail(
f"CAS device should be in inactive state. Actual state: {core_status}."
)
with TestRun.step(
"Try stopping cache without ‘no data flush’ option, verify that "
"operation was blocked and proper message is displayed."):
try_stop_incomplete_cache(cache)
with TestRun.step(
"Stop cache with 'no data flush' option and plug missing core device."
):
cache.stop(no_data_flush=True)
plug_device.plug()
def get_test_configuration():
config = InitConfig.create_init_config_from_running_configuration()
devices = get_block_devices_list()
return config, devices
