def test_stress_start(cache_mode):
"""
title: Stress test for starting and stopping cache.
description: Validate the ability of CAS to start and stop cache in the loop.
pass_criteria:
- No system crash while starting and stopping cache in the loop.
- Cache starts and stops successfully.
"""
with TestRun.step("Prepare cache and core."):
cache_dev, core_dev = prepare()
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Start and stop CAS {iterations_per_config} times."):
with TestRun.step("Start cache."):
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
with TestRun.step("Add core."):
cache.add_core(core_dev)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def pytest_runtest_teardown():
"""
This method is executed always in the end of each test, even if it fails or raises exception in
prepare stage.
"""
TestRun.LOGGER.end_all_groups()
with TestRun.LOGGER.step("Cleanup after test"):
try:
if TestRun.executor:
if not TestRun.executor.is_active():
TestRun.executor.wait_for_connection()
Udev.enable()
kill_all_io()
unmount_cas_devices()
if installer.check_if_installed():
casadm.remove_all_detached_cores()
casadm.stop_all_caches()
from api.cas.init_config import InitConfig
InitConfig.create_default_init_config()
DeviceMapper.remove_all()
except Exception as ex:
TestRun.LOGGER.warning(f"Exception occurred during platform cleanup.\n"
f"{str(ex)}\n{traceback.format_exc()}")
TestRun.LOGGER.end()
for dut in TestRun.duts:
with TestRun.use_dut(dut):
if TestRun.executor:
os.makedirs(os.path.join(TestRun.LOGGER.base_dir, "dut_info", dut.ip),
exist_ok=True)
TestRun.LOGGER.get_additional_logs()
Log.destroy()
TestRun.teardown()
def test_stop_cache_during_io():
"""
title: Test for stopping cache during IO.
description: |
Creating CAS device, running fio on it and checking
if cache can be stopped during IO operations.
pass_criteria:
- Cache is not stopped.
"""
with TestRun.step("Start cache and add core"):
cache, core = prepare()
with TestRun.step("Running 'fio'"):
fio = (
Fio()
.create_command()
.io_engine(IoEngine.libaio)
.block_size(Size(4, Unit.KibiByte))
.read_write(ReadWrite.randrw)
.target(f"{core.system_path}")
.direct(1)
.run_time(timedelta(minutes=4))
.time_based()
)
fio_pid = fio.run_in_background()
time.sleep(10)
with TestRun.step("Try to stop cache during 'fio'"):
TestRun.executor.run_expect_fail(cli.stop_cmd(f"{cache.cache_id}"))
with TestRun.step("Stopping 'fio'"):
TestRun.executor.kill_process(fio_pid)
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
def test_stress_load(cache_mode):
"""
title: Stress test for stopping and loading CAS device.
description: |
Validate the ability of the CAS to load and stop cache in the loop
using different cache modes.
pass_criteria:
- No system crash while stop and load cache in the loop.
- CAS device loads successfully.
"""
with TestRun.step("Prepare cache and core."):
cache_dev, core_dev = prepare()
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
casadm.add_core(cache, core_dev)
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Stop cache and load it {iterations_per_config} times."):
with TestRun.step("Stop cache."):
casadm.stop_cache(cache.cache_id)
if len(casadm_parser.get_caches()) != 0:
TestRun.fail("Cache did not stop successfully.")
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Cache did not load successfully - wrong number of caches: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.LOGGER.error(f"Cache loaded with wrong cores count: {cores_count}.")
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def pytest_runtest_teardown():
"""
This method is executed always in the end of each test, even if it fails or raises exception in
prepare stage.
"""
if TestRun.outcome == "skipped":
return
TestRun.LOGGER.end_all_groups()
with TestRun.LOGGER.step("Cleanup after test"):
try:
if TestRun.executor:
if TestRun.executor.is_active():
TestRun.executor.wait_for_connection()
Udev.enable()
unmount_cas_devices()
casadm.stop_all_caches()
except Exception as ex:
TestRun.LOGGER.warning(
f"Exception occured during platform cleanup.\n"
f"{str(ex)}\n{traceback.format_exc()}")
if 'test_wrapper' in sys.modules:
try:
test_wrapper.cleanup()
except Exception as ex:
TestRun.LOGGER.warning(
f"Exception occured during test wrapper cleanup.\n{str(ex)}"
f"\n{traceback.format_exc()}")
TestRun.LOGGER.end()
if TestRun.executor:
TestRun.LOGGER.get_additional_logs()
Log.destroy()
def test_another_cache_with_same_id():
"""
title: Test for creating another cache device with the same ID.
description: |
Checking if adding another cache device and setting
the same cache ID as the previous one fails.
pass_criteria:
- No additional cache device added.
"""
with TestRun.step("Start cache with ID = 1"):
cache_dev_1 = TestRun.disks["cache_1"]
cache_dev_1.create_partitions([Size(2, Unit.GibiByte)])
TestRun.executor.run_expect_success(
cli.start_cmd(cache_dev_1.partitions[0].system_path,
cache_id="1",
force=True))
with TestRun.step("Try to start another cache with the same ID = 1"):
cache_dev_2 = TestRun.disks["cache_2"]
cache_dev_2.create_partitions([Size(2, Unit.GibiByte)])
TestRun.executor.run_expect_fail(
cli.start_cmd(cache_dev_2.partitions[0].system_path,
cache_id="1",
force=True))
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
def test_stress_reload_cache(cache_mode):
"""
title: Stress test for reloading cache with simple data integrity check.
description: |
Validate the ability of CAS to reload cache in the loop
with no data corruption.
pass_criteria:
- No system crash while reloading cache.
- CAS device loads successfully.
- No data corruption.
"""
with TestRun.step("Prepare cache and core. Create test file and count it's checksum."):
cache, core, md5_before_load, size_before_load, permissions_before_load = \
prepare_with_file_creation(cache_mode)
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Stop and load cache {iterations_per_config} times."):
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Check md5 of test file."):
check_files(core, size_before_load, permissions_before_load, md5_before_load)
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def base_prepare(item):
with TestRun.LOGGER.step("Cleanup before test"):
Udev.enable()
kill_all_io()
if installer.check_if_installed():
try:
unmount_cas_devices()
casadm.stop_all_caches()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
for disk in TestRun.dut.disks:
disk.umount_all_partitions()
if not create_partition_table(disk, PartitionTable.gpt):
TestRun.exception(f"Failed to remove partitions from {disk}")
if get_force_param(
item) and not TestRun.plugins['opencas'].already_updated:
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.install_opencas()
TestRun.plugins['opencas'].already_updated = True
from api.cas import init_config
init_config.create_default_init_config()
TestRun.LOGGER.add_build_info(f'Commit hash:')
TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}")
TestRun.LOGGER.add_build_info(f'Commit message:')
TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}')
def base_prepare(item):
with TestRun.LOGGER.step("Cleanup before test"):
TestRun.executor.run("pkill --signal=SIGKILL fsck")
Udev.enable()
kill_all_io()
DeviceMapper.remove_all()
if installer.check_if_installed():
try:
from api.cas.init_config import InitConfig
InitConfig.create_default_init_config()
unmount_cas_devices()
casadm.stop_all_caches()
casadm.remove_all_detached_cores()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
for disk in TestRun.dut.disks:
disk.umount_all_partitions()
create_partition_table(disk, PartitionTable.gpt)
if get_force_param(item) and not TestRun.usr.already_updated:
installer.rsync_opencas_sources()
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.rsync_opencas_sources()
installer.set_up_opencas()
TestRun.usr.already_updated = True
TestRun.LOGGER.add_build_info(f'Commit hash:')
TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}")
TestRun.LOGGER.add_build_info(f'Commit message:')
TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}')
def test_cleaning_policies_in_write_back(cleaning_policy):
"""
title: Test for cleaning policy operation in Write-Back cache mode.
description: |
Check if ALRU, NOP and ACP cleaning policies preserve their
parameters when changed and if they flush dirty data properly
in Write-Back cache mode.
pass_criteria:
- Flush parameters preserve their values when changed.
- Dirty data is flushed or not according to the policy used.
"""
with TestRun.step("Partition cache and core devices"):
cache_dev, core_dev = storage_prepare()
Udev.disable()
with TestRun.step(
f"Start cache in Write-Back mode with {cleaning_policy} cleaning policy"
):
cache = casadm.start_cache(cache_dev.partitions[0],
CacheMode.WB,
force=True)
set_cleaning_policy_and_params(cache, cleaning_policy)
with TestRun.step("Check for running CAS cleaner"):
if TestRun.executor.run(
f"pgrep {cas_cleaner_process_name}").exit_code != 0:
TestRun.fail("CAS cleaner process is not running!")
with TestRun.step(f"Add {cores_count} cores to the cache"):
core = []
for i in range(cores_count):
core.append(cache.add_core(core_dev.partitions[i]))
with TestRun.step("Run 'fio'"):
fio = fio_prepare()
for i in range(cores_count):
fio.add_job().target(core[i].path)
fio.run()
time.sleep(3)
core_writes_before_wait_for_cleaning = (
cache.get_statistics().block_stats.core.writes)
with TestRun.step(f"Wait {time_to_wait} seconds"):
time.sleep(time_to_wait)
with TestRun.step("Check write statistics for core device"):
core_writes_after_wait_for_cleaning = (
cache.get_statistics().block_stats.core.writes)
check_cleaning_policy_operation(
cleaning_policy,
core_writes_before_wait_for_cleaning,
core_writes_after_wait_for_cleaning,
)
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
Udev.enable()
def prepare_and_cleanup(request):
"""
This fixture returns the dictionary, which contains DUT ip, IPMI, spider, list of disks.
This fixture also returns the executor of commands
"""
# There should be dut config file added to config package and
# pytest should be executed with option --dut-config=conf_name'.
#
# 'ip' field should be filled with valid IP string to use remote ssh executor
# or it should be commented out when user want to execute tests on local machine
#
# User can also have own test wrapper, which runs test prepare, cleanup, etc.
# Then in the config/configuration.py file there should be added path to it:
# test_wrapper_dir = 'wrapper_path'
LOGGER.info(f"**********Test {request.node.name} started!**********")
try:
dut_config = importlib.import_module(f"config.{request.config.getoption('--dut-config')}")
except:
dut_config = None
if os.path.exists(c.test_wrapper_dir):
if hasattr(dut_config, 'ip'):
try:
IP(dut_config.ip)
except ValueError:
raise Exception("IP address from configuration file is in invalid format.")
TestProperties.dut = Dut(test_wrapper.prepare(request, dut_config))
elif dut_config is not None:
if hasattr(dut_config, 'ip'):
try:
IP(dut_config.ip)
if hasattr(dut_config, 'user') and hasattr(dut_config, 'password'):
executor = SshExecutor(dut_config.ip, dut_config.user, dut_config.password)
TestProperties.executor = executor
else:
raise Exception("There is no credentials in config file.")
if hasattr(dut_config, 'disks'):
TestProperties.dut = Dut({'ip': dut_config.ip, 'disks': dut_config.disks})
else:
TestProperties.dut = Dut(
{'ip': dut_config.ip, 'disks': disk_finder.find_disks()})
except ValueError:
raise Exception("IP address from configuration file is in invalid format.")
elif hasattr(dut_config, 'disks'):
TestProperties.executor = LocalExecutor()
TestProperties.dut = Dut({'disks': dut_config.disks})
else:
TestProperties.executor = LocalExecutor()
TestProperties.dut = Dut({'disks': disk_finder.find_disks()})
else:
raise Exception(
"There is neither configuration file nor test wrapper attached to tests execution.")
yield
TestProperties.LOGGER.info("Test cleanup")
casadm.stop_all_caches()
if os.path.exists(c.test_wrapper_dir):
test_wrapper.cleanup(TestProperties.dut)
def base_prepare(item):
with TestRun.LOGGER.step("Cleanup before test"):
TestRun.executor.run("pkill --signal=SIGKILL fsck")
Udev.enable()
kill_all_io()
DeviceMapper.remove_all()
if installer.check_if_installed():
try:
from api.cas.init_config import InitConfig
InitConfig.create_default_init_config()
unmount_cas_devices()
casadm.stop_all_caches()
casadm.remove_all_detached_cores()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
raids = Raid.discover()
for raid in raids:
# stop only those RAIDs, which are comprised of test disks
if all(map(lambda device:
any(map(lambda disk_path:
disk_path in device.get_device_id(),
[bd.get_device_id() for bd in TestRun.dut.disks])),
raid.array_devices)):
raid.umount_all_partitions()
raid.remove_partitions()
raid.stop()
for device in raid.array_devices:
Mdadm.zero_superblock(os.path.join('/dev', device.get_device_id()))
Udev.settle()
for disk in TestRun.dut.disks:
disk_serial = get_disk_serial_number(disk.path)
if disk.serial_number != disk_serial:
raise Exception(
f"Serial for {disk.path} doesn't match the one from the config."
f"Serial from config {disk.serial_number}, actual serial {disk_serial}"
)
disk.umount_all_partitions()
Mdadm.zero_superblock(os.path.join('/dev', disk.get_device_id()))
TestRun.executor.run_expect_success("udevadm settle")
disk.remove_partitions()
create_partition_table(disk, PartitionTable.gpt)
if get_force_param(item) and not TestRun.usr.already_updated:
installer.rsync_opencas_sources()
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.rsync_opencas_sources()
installer.set_up_opencas()
TestRun.usr.already_updated = True
TestRun.LOGGER.add_build_info(f'Commit hash:')
TestRun.LOGGER.add_build_info(f"{git.get_current_commit_hash()}")
TestRun.LOGGER.add_build_info(f'Commit message:')
TestRun.LOGGER.add_build_info(f'{git.get_current_commit_message()}')
def test_one_core_release(cache_mode):
"""
title: Test if OpenCAS dynamically allocates space according to core devices needs.
description: |
When one or more core devices are unused in a single cache instance all blocks
previously occupied should be available to other core devices.
Test is without pass through mode.
pass_criteria:
- No system crash.
- The remaining core is able to use cache.
- OpenCAS frees blocks occupied by unused core and allocates it to the remaining core.
"""
with TestRun.step("Prepare two cache and one core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte)] * 2)
core_part1 = core_dev.partitions[0]
core_part2 = core_dev.partitions[1]
Udev.disable()
with TestRun.step("Start cache"):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
with TestRun.step("Add both core devices to cache."):
core1 = cache.add_core(core_part1)
core2 = cache.add_core(core_part2)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 2:
TestRun.fail(f"Expected cores count: 2; Actual cores count: {cores_count}.")
with TestRun.step("Change sequential cutoff policy to 'never'."):
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Fill cache with pages from the first core."):
dd_builder(cache_mode, core1, cache.size).run()
core1_occupied_blocks_before = core1.get_occupancy()
with TestRun.step("Check if the remaining core is able to use cache."):
dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run()
core1_occupied_blocks_after = core1.get_occupancy()
with TestRun.step("Check if occupancy from the first core is removed from cache."):
# The first core's occupancy should be lower than cache's occupancy
# by the value of the remaining core's occupancy because cache
# should reallocate blocks from unused core to used core.
if core1_occupied_blocks_after >= core1_occupied_blocks_before \
or cache.get_occupancy() <= core1_occupied_blocks_after \
or not float(core2.get_occupancy().get_value()) > 0:
TestRun.LOGGER.error("Blocks previously occupied by the first core aren't released.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_load_cache_with_mounted_core(cache_mode):
"""
title: Fault injection test for adding mounted core on cache load.
description: |
Negative test of the ability of CAS to add to cache while its loading
core device which is mounted.
pass_criteria:
- No system crash while loading cache.
- Adding mounted core while loading cache fails.
"""
with TestRun.step("Prepare cache and core devices. Start CAS."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_part = core_dev.partitions[0]
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Add core device with xfs filesystem to cache and mount it."):
core_part.create_filesystem(Filesystem.xfs)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step(f"Create test file in mount point of exported object and check its md5 sum."):
test_file = fs_utils.create_random_test_file(test_file_path)
test_file_md5_before = test_file.md5sum()
with TestRun.step("Unmount core device."):
core.unmount()
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
with TestRun.step("Mount core device."):
core_part.mount(mount_point)
with TestRun.step("Try to load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1 Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(f"Expected cores count: 0; Actual cores count: {cores_count}.")
with TestRun.step("Check md5 sum of test file again."):
if test_file_md5_before != test_file.md5sum():
TestRun.LOGGER.error("Md5 sum of test file is different.")
core_part.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_purge(purge_target):
"""
title: Call purge without and with `--script` switch
description: |
Check if purge is called only when `--script` switch is used.
pass_criteria:
- casadm returns an error when `--script` is missing
- cache is wiped when purge command is used properly
"""
with TestRun.step("Prepare devices"):
cache_device = TestRun.disks["cache"]
core_device = TestRun.disks["core"]
cache_device.create_partitions([Size(500, Unit.MebiByte)])
core_device.create_partitions([Size(500, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
core_device = core_device.partitions[0]
with TestRun.step("Prepare cache instance"):
cache = casadm.start_cache(cache_device, force=True)
core = casadm.add_core(cache, core_device)
with TestRun.step("Trigger IO to prepared cache instance"):
dd = (Dd().input("/dev/zero").output(core.path).count(100).block_size(
Size(1, Unit.Blocks512)).oflag("direct"))
dd.run()
sync()
with TestRun.step(
f"Try to call purge-{purge_target} without `--script` switch"):
original_occupancy = cache.get_statistics().usage_stats.occupancy
purge_params = f"--cache-id {cache.cache_id} "
if purge_target == "core":
purge_params += f"--core-id {core.core_id}"
TestRun.executor.run_expect_fail(
f"casadm --purge-{purge_target} {purge_params}")
if cache.get_statistics().usage_stats.occupancy != original_occupancy:
TestRun.fail(
f"Purge {purge_target} should not be possible to use without `--script` switch!"
)
with TestRun.step(
f"Try to call purge-{purge_target} with `--script` switch"):
TestRun.executor.run_expect_success(
f"casadm --script --purge-{purge_target} {purge_params}")
if cache.get_statistics().usage_stats.occupancy.get_value() != 0:
TestRun.fail(
f"{cache.get_statistics().usage_stats.occupancy.get_value()}")
TestRun.fail(
f"Purge {purge_target} should invalidate all cache lines!")
with TestRun.step(f"Stop cache"):
casadm.stop_all_caches()
def base_prepare():
LOGGER.info("Base test prepare")
LOGGER.info(f"DUT info: {TestProperties.dut}")
LOGGER.info("Removing partitions")
for disk in TestProperties.dut.disks:
disk_utils.remove_partitions(disk)
if get_force_param() is not "False" and not hasattr(c, "already_updated"):
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.install_opencas()
c.already_updated = True # to skip reinstall every test
casadm.stop_all_caches()
def test_stress_add_remove_core(cache_mode):
"""
title: Stress test for adding and removing core.
description: Validate the ability of CAS to add and remove core in the loop.
pass_criteria:
- No system crash while adding and removing core.
- Core is added and removed successfully.
- No data corruption.
"""
with TestRun.step(
"Prepare cache and core. Create test file and count its checksum."
):
cache, core, file, file_md5sum_before = prepare_with_file_creation(
cache_mode)
for _ in TestRun.iteration(
range(0, iterations_per_config),
f"Add and remove core {iterations_per_config} times."):
with TestRun.step("Remove core."):
core.remove_core()
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(
f"Expected cores count: 0; Actual cores count: {cores_count}."
)
with TestRun.step("Add core."):
core = cache.add_core(core.core_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}."
)
with TestRun.step("Check md5 sum of test file."):
core.mount(mount_point)
file_md5sum_after = file.md5sum()
if file_md5sum_after != file_md5sum_before:
TestRun.LOGGER.error("Md5 sum of test file is different.")
core.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def test_stress_service(cache_mode):
"""
title: Stress test for starting and stopping CAS service.
description: |
Validate the ability of CAS to restart CAS service
and load CAS device in the loop.
pass_criteria:
- No system crash while restarting CAS service or loading cache.
- CAS service restarts with no errors.
- CAS device loads successfully.
"""
with TestRun.step("Prepare cache and core."):
cache_dev, core_dev = prepare()
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_dev, cache_mode, force=True)
casadm.add_core(cache, core_dev)
for _ in TestRun.iteration(
range(0, iterations_per_config),
f"Stop and start CAS service {iterations_per_config} times."):
with TestRun.step(
"Create CAS init config based on current running CAS configuration."
):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Stop CAS service."):
casctl.stop()
with TestRun.step("Check if service stopped successfully."):
if len(casadm_parser.get_caches()) != 0:
TestRun.fail(
"There are still running caches after stopping service.")
if len(casadm_parser.get_cores(cache.cache_id)) != 0:
TestRun.fail(
"There are still running cores after stopping service.")
with TestRun.step("Start CAS service."):
casctl.start()
time.sleep(1) # Time for CAS devices to start
with TestRun.step("Check if CAS configuration loaded successfully."):
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(
f"Expected cores count: 1; Actual cores count: {cores_count}."
)
with TestRun.step("Stop caches and create default init config file."):
casadm.stop_all_caches()
InitConfig.create_default_init_config()
def test_cache_mode_switching_during_io(cache_mode_1, cache_mode_2, flush, io_mode):
"""
title: Test for dynamic cache mode switching during IO.
description: |
Validate the ability of CAS to switch cache modes
during working IO on CAS device.
pass_criteria:
- Cache mode is switched without errors.
"""
with TestRun.step("Partition cache and core devices"):
cache_dev, core_dev = storage_prepare()
with TestRun.step(f"Start cache in {cache_mode_1} mode"):
cache = casadm.start_cache(cache_dev, cache_mode_1, force=True)
with TestRun.step("Add core to the cache"):
core = cache.add_core(core_dev)
with TestRun.step("Run 'fio'"):
fio = (
fio_prepare(core, io_mode)
.verify(VerifyMethod.sha1)
.run_time(timedelta(minutes=4))
.time_based()
)
fio_pid = fio.run_in_background()
time.sleep(5)
with TestRun.step(
f"Change cache mode to {cache_mode_2} with flush cache option set to: {flush}"
):
cache_mode_switch_output = cache.set_cache_mode(cache_mode_2, flush)
if cache_mode_switch_output.exit_code != 0:
TestRun.fail("Cache mode switch failed!")
with TestRun.step(f"Check if cache mode has switched properly during IO"):
cache_mode_after_switch = cache.get_cache_mode()
if cache_mode_after_switch != cache_mode_2:
TestRun.fail(
f"Cache mode did not switch properly! "
f"Cache mode after switch is: {cache_mode_after_switch}, "
f"should be: {cache_mode_2}"
)
with TestRun.step("Stop 'fio'"):
TestRun.executor.kill_process(fio_pid)
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
def test_add_cached_core(cache_mode):
"""
title: Fault injection test for adding already used core to a cache.
description: |
Negative test of the ability to add the core to the cache twice to the same cache
and while the core is already used by the another cache instance.
pass_criteria:
- No system crash.
- Adding already used core to another cache instance fails.
- The same core device cannot be used twice in one cache instance.
"""
with TestRun.step("Prepare two caches and one core device."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions(
[Size(2, Unit.GibiByte),
Size(2, Unit.GibiByte)])
cache_part1 = cache_dev.partitions[0]
cache_part2 = cache_dev.partitions[1]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_part = core_dev.partitions[0]
with TestRun.step("Start the first cache instance"):
cache1 = casadm.start_cache(cache_part1, cache_mode, force=True)
with TestRun.step("Add core device to first cache instance."):
core = cache1.add_core(core_part)
with TestRun.step("Start the second cache instance"):
cache2 = casadm.start_cache(cache_part2, cache_mode, force=True)
with TestRun.step(
"Try adding the same core device to the second cache instance."):
output = TestRun.executor.run_expect_fail(
cli.add_core_cmd(cache_id=str(cache2.cache_id),
core_dev=str(core_part.system_path),
core_id=str(core.core_id)))
cli_messages.check_msg(output, cli_messages.add_cached_core)
with TestRun.step(
"Try adding the same core device to the same cache for the second time."
):
output = TestRun.executor.run_expect_fail(
cli.add_core_cmd(cache_id=str(cache1.cache_id),
core_dev=str(core_part.system_path)))
cli_messages.check_msg(output, cli_messages.add_cached_core)
with TestRun.step("Stop caches."):
casadm.stop_all_caches()
def test_stress_reload_module(cache_mode):
"""
title: Stress test for reloading CAS modules.
description: Validate the ability of CAS to reload modules in the loop.
pass_criteria:
- No system crash while reloading CAS modules.
- CAS modules reloads with no errors.
- No data corruption.
"""
with TestRun.step("Prepare cache and core. Create test file and count its checksum."):
cache, core, file, file_md5sum_before = prepare_with_file_creation(cache_mode)
with TestRun.step("Save current cache configuration."):
cache_config = cache.get_cache_config()
for _ in TestRun.iteration(range(0, iterations_per_config),
f"Reload CAS modules and check loaded "
f"cache configuration {iterations_per_config} times."):
with TestRun.step("Stop cache."):
cache.stop()
caches_count = len(casadm_parser.get_caches())
if caches_count != 0:
TestRun.fail(f"Expected caches count: 0; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 0:
TestRun.fail(f"Expected cores count: 0; Actual cores count: {cores_count}.")
with TestRun.step("Reload CAS modules."):
cas_module.reload_all_cas_modules()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache.cache_device)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(f"Expected caches count: 1; Actual caches count: {caches_count}.")
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 1:
TestRun.fail(f"Expected cores count: 1; Actual cores count: {cores_count}.")
with TestRun.step("Validate cache configuration."):
if cache.get_cache_config() != cache_config:
TestRun.fail("Cache configuration is different than before reloading modules.")
with TestRun.step("Check md5 sum of test file."):
core.mount(mount_point)
file_md5sum_after = file.md5sum()
if file_md5sum_after != file_md5sum_before:
TestRun.LOGGER.error("Md5 sum of test file is different.")
core.unmount()
with TestRun.step("Stop all caches."):
casadm.stop_all_caches()
def prepare_and_cleanup(request):
"""
This fixture returns the dictionary, which contains DUT ip, IPMI, spider, list of disks.
This fixture also returns the executor of commands
"""
# There should be dut config file added to config package and
# pytest should be executed with option --dut-config=conf_name'.
#
# 'ip' field should be filled with valid IP string to use remote ssh executor
# or it should be commented out when user want to execute tests on local machine
#
# User can also have own test wrapper, which runs test prepare, cleanup, etc.
# Then in the config/configuration.py file there should be added path to it:
# test_wrapper_dir = 'wrapper_path'
try:
with open(request.config.getoption('--dut-config')) as cfg:
dut_config = yaml.safe_load(cfg)
except Exception:
dut_config = {}
if 'test_wrapper' in sys.modules:
if 'ip' in dut_config:
try:
IP(dut_config['ip'])
except ValueError:
raise Exception(
"IP address from configuration file is in invalid format.")
dut_config = test_wrapper.prepare(request.param, dut_config)
TestRun.prepare(dut_config)
TestRun.plugins['opencas'] = {'already_updated': False}
TestRun.LOGGER.info(
f"**********Test {request.node.name} started!**********")
yield
TestRun.LOGGER.info("Test cleanup")
Udev.enable()
unmount_cas_devices()
casadm.stop_all_caches()
if 'test_wrapper' in sys.modules:
test_wrapper.cleanup()
def test_stats_values():
"""
title: Check for proper statistics values.
description: |
Check if CAS displays proper usage, request, block and error statistics values
for core devices in every cache mode - at the start, after IO and after cache
reload. Also check if core's statistics match cache's statistics.
pass_criteria:
- Usage, request, block and error statistics have proper values.
- Core's statistics match cache's statistics.
"""
with TestRun.step("Partition cache and core devices"):
cache_dev, core_dev = storage_prepare()
Udev.disable()
with TestRun.step(
f"Start {caches_count} caches (one for every cache mode) "
f"and add {cores_per_cache} cores per cache"
):
caches, cores = cache_prepare(cache_dev, core_dev)
with TestRun.step("Check initial statistics values for each core"):
check_stats_initial(caches, cores)
with TestRun.step("Run 'fio'"):
fio = fio_prepare()
for i in range(caches_count):
for j in range(cores_per_cache):
fio.add_job().target(cores[i][j].path)
fio.run()
sleep(3)
with TestRun.step("Check statistics values after IO"):
check_stats_after_io(caches, cores)
with TestRun.step("Check if cache's statistics match core's statistics"):
check_stats_sum(caches, cores)
with TestRun.step("Stop and load caches back"):
casadm.stop_all_caches()
caches = cache_load(cache_dev)
with TestRun.step("Check statistics values after reload"):
check_stats_after_io(caches, cores, after_reload=True)
def test_load_corrupted():
"""
title: Standby-load corrupted metadata
description: |
Try to load standby instance from corrupted metadata
pass_criteria:
- Kernel panic doesn't occur
"""
with TestRun.step("Prepare devices for the cache and core."):
cache_device = TestRun.disks["cache"]
cache_device.create_partitions([Size(200, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
core_device = TestRun.disks["core"]
core_device.create_partitions([Size(500, Unit.MebiByte)])
core_device = core_device.partitions[0]
with TestRun.step("Prepare metadata dump"):
cache_id = 1
cls = CacheLineSize.LINE_32KiB
md_dump = prepare_md_dump(cache_device, core_device, cls, cache_id)
for offset in get_offsets_to_corrupt(md_dump.size, block_size):
with TestRun.step(
f"Corrupt {block_size} on the offset {offset*block_size}"):
corrupted_md = prepare_corrupted_md(md_dump, offset, block_size)
with TestRun.step(
f"Copy corrupted metadata to the cache-to-be device"):
Dd().input(corrupted_md.full_path).output(cache_device.path).run()
sync()
with TestRun.step("Try to load cache instance"):
output = TestRun.executor.run(
standby_load_cmd(cache_dev=cache_device.path))
with TestRun.step("Per iteration cleanup"):
if output.exit_code:
casadm.stop_all_caches()
corrupted_md.remove(force=True, ignore_errors=True)
with TestRun.step("Test cleanup"):
md_dump.remove()
def test_stop_cache_with_mounted_partition(cache_mode):
"""
title: Fault injection test for removing core and stopping cache with mounted core.
description: |
Negative test of the ability of CAS to remove core and stop cache while core
is still mounted.
pass_criteria:
- No system crash.
- Unable to stop cache when partition is mounted.
- Unable to remove core when partition is mounted.
"""
with TestRun.step("Prepare cache and core devices. Start CAS."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(4, Unit.GibiByte)])
core_part = core_dev.partitions[0]
cache = casadm.start_cache(cache_part, cache_mode, force=True)
with TestRun.step("Add core device with xfs filesystem and mount it."):
core_part.create_filesystem(Filesystem.xfs)
core = cache.add_core(core_part)
core.mount(mount_point)
with TestRun.step("Try to remove core from cache."):
output = TestRun.executor.run_expect_fail(
cli.remove_core_cmd(cache_id=str(cache.cache_id),
core_id=str(core.core_id)))
cli_messages.check_stderr_msg(output, cli_messages.remove_mounted_core)
with TestRun.step("Try to stop CAS."):
output = TestRun.executor.run_expect_fail(
cli.stop_cmd(cache_id=str(cache.cache_id)))
cli_messages.check_stderr_msg(output,
cli_messages.stop_cache_mounted_core)
with TestRun.step("Unmount core device."):
core.unmount()
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def test_remove_multilevel_core():
"""
title: Test of the ability to remove a core used in a multilevel cache.
description: |
Negative test if OpenCAS does not allow to remove a core when the related exported object
is used as a core device for another cache instance.
pass_criteria:
- No system crash.
- OpenCAS does not allow removing a core used in a multilevel cache instance.
"""
with TestRun.step("Prepare two devices for cache and one for core."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(512, Unit.MebiByte)] * 2)
cache_part1 = cache_dev.partitions[0]
cache_part2 = cache_dev.partitions[1]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(1, Unit.GibiByte)])
core_dev = core_dev.partitions[0]
with TestRun.step("Start the first cache instance"):
cache1 = casadm.start_cache(cache_part1, force=True)
with TestRun.step("Add a core device to the first cache instance."):
core1 = cache1.add_core(core_dev)
with TestRun.step("Start the second cache instance"):
cache2 = casadm.start_cache(cache_part2, force=True)
with TestRun.step("Add the first cache's exported object as a core "
"to the second cache instance."):
cache2.add_core(core1)
with TestRun.step("Try to remove core from the first level cache."):
output = TestRun.executor.run_expect_fail(
cli.remove_core_cmd(cache_id=str(cache1.cache_id),
core_id=str(core1.core_id),
force=True))
cli_messages.check_stderr_msg(output,
cli_messages.remove_multilevel_core)
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
def base_prepare():
LOGGER.info("Base test prepare")
LOGGER.info(f"DUT info: {TestProperties.dut}")
if installer.check_if_installed():
try:
unmount_cas_devices()
casadm.stop_all_caches()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
for disk in TestProperties.dut.disks:
if disk.is_mounted():
disk.unmount()
disk.remove_partitions()
if get_force_param() is not "False" and not hasattr(c, "already_updated"):
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.install_opencas()
c.already_updated = True # to skip reinstall every test
def test_cas_init_with_changed_mode(cache_mode_pair):
"""
title: Check starting cache in other cache mode by initializing OpenCAS service from config.
description: |
Start cache, create config based on running configuration but with another cache mode,
reinitialize OpenCAS service with '--force' option and check if cache defined
in config file starts properly.
Check all cache modes.
pass_criteria:
- Cache starts with attached core
- Cache starts in mode saved in configuration file.
"""
with TestRun.step("Prepare partitions for cache and core."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(200, Unit.MebiByte)])
cache_part = cache_dev.partitions[0]
core_dev = TestRun.disks['core']
core_dev.create_partitions([Size(400, Unit.MebiByte)])
core_part = core_dev.partitions[0]
with TestRun.step(
f"Start cache in the {cache_mode_pair[0]} mode and add core."):
cache = casadm.start_cache(cache_part, cache_mode_pair[0], force=True)
core = cache.add_core(core_part)
with TestRun.step(
f"Create the configuration file with a different cache mode ({cache_mode_pair[1]})"
):
init_conf = InitConfig()
init_conf.add_cache(cache.cache_id, cache.cache_device,
cache_mode_pair[1])
init_conf.add_core(cache.cache_id, core.core_id, core.core_device)
init_conf.save_config_file()
with TestRun.step("Reinitialize OpenCAS service with '--force' option."):
casadm.stop_all_caches()
casctl.init(True)
with TestRun.step(
"Check if cache started in correct mode with core attached."):
validate_cache(cache_mode_pair[1])
def base_prepare():
TestRun.LOGGER.info("Base test prepare")
TestRun.LOGGER.info(f"DUT info: {TestRun.dut}")
Udev.enable()
kill_all_io()
if installer.check_if_installed():
try:
unmount_cas_devices()
casadm.stop_all_caches()
except Exception:
pass # TODO: Reboot DUT if test is executed remotely
if get_force_param(
) is not "False" and not TestRun.plugins['opencas']['already_updated']:
installer.reinstall_opencas()
elif not installer.check_if_installed():
installer.install_opencas()
TestRun.plugins['opencas']['already_updated'] = True
def test_another_core_with_same_id():
"""
title: Test for creating another core device with the same ID.
description: |
Checking if adding another core device and setting
the same core ID as the previous one fails.
pass_criteria:
- No additional core device added.
"""
with TestRun.step("Start cache device"):
cache_dev = TestRun.disks["cache"]
cache_dev.create_partitions([Size(2, Unit.GibiByte)])
cache = casadm.start_cache(cache_dev.partitions[0], force=True)
with TestRun.step("Add core with ID = 1"):
core_dev_1 = TestRun.disks["core_1"]
core_dev_1.create_partitions([Size(1, Unit.GibiByte)])
TestRun.executor.run_expect_success(
cli.add_core_cmd(
cache_id=f"{cache.cache_id}",
core_dev=f"{core_dev_1.partitions[0].path}",
core_id="1",
)
)
with TestRun.step("Try to add another core with the same ID = 1"):
core_dev_2 = TestRun.disks["core_2"]
core_dev_2.create_partitions([Size(1, Unit.GibiByte)])
TestRun.executor.run_expect_fail(
cli.add_core_cmd(
cache_id=f"{cache.cache_id}",
core_dev=f"{core_dev_2.partitions[0].path}",
core_id="1",
)
)
with TestRun.step("Stop all caches"):
casadm.stop_all_caches()
