def test_cli_add_remove_default_value(shortcut):
cache_device = TestRun.disks['cache']
cache_device.create_partitions([Size(50, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
cache = casadm.start_cache(cache_device, shortcut=shortcut, force=True)
core_device = TestRun.disks['core']
casadm.add_core(cache, core_device, shortcut=shortcut)
caches = casadm_parser.get_caches()
if len(caches[0].get_core_devices()) != 1:
TestRun.fail("One core should be present in cache")
if caches[0].get_core_devices(
)[0].core_device.system_path != core_device.system_path:
TestRun.fail("Core path should equal to path of core added")
casadm.remove_core(cache.cache_id, 1, shortcut=shortcut)
caches = casadm_parser.get_caches()
if len(caches) != 1:
TestRun.fail("One cache should be present still after removing core")
if len(caches[0].get_core_devices()) != 0:
TestRun.fail("No core devices should be present after removing core")
casadm.stop_cache(cache_id=cache.cache_id, shortcut=shortcut)
output = casadm.list_caches(shortcut=shortcut)
caches = casadm_parser.get_caches()
if len(caches) != 0:
TestRun.fail("No cache should be present after stopping the cache")
if output.stdout != "No caches running":
TestRun.fail(
f"Invalid message, expected 'No caches running', got {output.stdout}"
)
def test_cli_add_remove_default_value(prepare_and_cleanup, shortcut):
prepare()
cache_device = next(disk for disk in TestProperties.dut.disks
if disk.disk_type == DiskType.optane)
casadm.start_cache(cache_device, shortcut=shortcut, force=True)
core_device = next(disk for disk in TestProperties.dut.disks
if disk.disk_type != DiskType.optane)
casadm.add_core(1, core_device, shortcut=shortcut)
caches = casadm.parse_list_caches()
assert len(caches["1"]["cores"]) == 1
assert caches["1"]["cores"]["1"]["path"] == core_device.system_path
casadm.remove_core(1, 1, shortcut=shortcut)
caches = casadm.parse_list_caches()
assert len(caches) == 1
assert len(caches["1"]["cores"]) == 0
casadm.stop_cache(cache_id=1, shortcut=shortcut)
output = casadm.list_caches(shortcut=shortcut)
caches = casadm.parse_list_caches()
assert len(caches) == 0
assert output.stdout == "No caches running"
def test_cli_start_stop_default_value(shortcut):
with TestRun.LOGGER.step("Prepare devices"):
cache_device = TestRun.disks['cache']
cache_device.create_partitions([Size(500, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
with TestRun.LOGGER.step("Start cache"):
casadm.start_cache(cache_device, shortcut=shortcut, force=True)
with TestRun.LOGGER.step("Check if cache started successfully"):
caches = casadm_parser.get_caches()
if len(caches) != 1:
TestRun.LOGGER.error(
f"There is wrong caches count found in OS: {len(caches)}")
if caches[0].cache_device.system_path != cache_device.system_path:
TestRun.LOGGER.error(f"Cache started using wrong device: "
f"{caches[0].cache_device.system_path}. "
f"Should be {cache_device.system_path}")
with TestRun.LOGGER.step("Stop cache"):
casadm.stop_cache(cache_id=caches[0].cache_id, shortcut=shortcut)
with TestRun.LOGGER.step("Check if cache stopped properly"):
output = casadm.list_caches(shortcut=shortcut)
caches = casadm_parser.get_caches()
if len(caches) != 0:
TestRun.LOGGER.error(
f"There is wrong caches count found in OS: {len(caches)}. "
f"Should be 0.")
if output.stdout != "No caches running":
TestRun.LOGGER.error(
"There is no 'No caches running' info in casadm -L output")
def get_cas_devices_dict():
device_list = list(csv.DictReader(casadm.list_caches(OutputFormat.csv).stdout.split('\n')))
devices = {"core_pool": [], "caches": {}, "cores": {}}
core_pool = False
prev_cache_id = -1
for device in device_list:
if device["type"] == "core pool":
core_pool = True
continue
if device["type"] == "cache":
core_pool = False
prev_cache_id = int(device["id"])
devices["caches"].update(
{
int(device["id"]): {
"device": device["disk"],
"status": device["status"],
}
}
)
elif device["type"] == "core":
core = {"device": device["disk"], "status": device["status"]}
if core_pool:
devices["core_pool"].append(core)
else:
core.update({"cache_id": prev_cache_id})
devices["cores"].update(
{(prev_cache_id, int(device["id"])): core}
)
return devices
def test_cli_add_remove_default_value(prepare_and_cleanup, shortcut):
prepare()
cache_device = next(disk for disk in TestRun.dut.disks
if disk.disk_type == DiskType.optane)
cache_device.create_partitions([Size(500, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
cache = casadm.start_cache(cache_device, shortcut=shortcut, force=True)
core_device = next(disk for disk in TestRun.dut.disks
if disk.disk_type != DiskType.optane)
casadm.add_core(cache, core_device, shortcut=shortcut)
caches = casadm_parser.get_caches()
assert len(caches[0].get_core_devices()) == 1
assert caches[0].get_core_devices(
)[0].core_device.system_path == core_device.system_path
casadm.remove_core(cache.cache_id, 1, shortcut=shortcut)
caches = casadm_parser.get_caches()
assert len(caches) == 1
assert len(caches[0].get_core_devices()) == 0
casadm.stop_cache(cache_id=cache.cache_id, shortcut=shortcut)
output = casadm.list_caches(shortcut=shortcut)
caches = casadm_parser.get_caches()
assert len(caches) == 0
assert output.stdout == "No caches running"
def test_cli_add_remove_default_value(shortcut):
cache_device = TestRun.disks['cache']
cache_device.create_partitions([Size(500, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
cache = casadm.start_cache(cache_device, shortcut=shortcut, force=True)
core_device = TestRun.disks['core']
casadm.add_core(cache, core_device, shortcut=shortcut)
caches = casadm_parser.get_caches()
assert len(caches[0].get_core_devices()) == 1
assert caches[0].get_core_devices(
)[0].core_device.system_path == core_device.system_path
casadm.remove_core(cache.cache_id, 1, shortcut=shortcut)
caches = casadm_parser.get_caches()
assert len(caches) == 1
assert len(caches[0].get_core_devices()) == 0
casadm.stop_cache(cache_id=cache.cache_id, shortcut=shortcut)
output = casadm.list_caches(shortcut=shortcut)
caches = casadm_parser.get_caches()
assert len(caches) == 0
assert output.stdout == "No caches running"
def test_remove_detached_cores():
"""
title: Validate removing core devices from core pool.
description: Validate that it is possible to remove core devices from core pool.
pass_criteria:
- No kernel error
- All core devices are correctly added after plugging core disk.
- All cores are successfully removed.
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 4)])
cache_dev = devices["cache"].partitions[0]
core_devs = devices["core"].partitions
plug_device = devices["core"]
with TestRun.step("Start cache and add four cores."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cores = []
for d in core_devs:
cores.append(cache.add_core(d))
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run random writes to all CAS devices."):
run_fio([c.system_path for c in cores])
with TestRun.step(
"Flush dirty data from two CAS devices and verify than other two contain "
"dirty data."):
for core in cores:
if core.core_id % 2 == 0:
core.flush_core()
if core.get_dirty_blocks() != Size.zero():
TestRun.fail("Failed to flush CAS device.")
elif core.get_dirty_blocks() == Size.zero():
TestRun.fail("There should be dirty data on CAS device.")
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug core device from system and plug it back."):
plug_device.unplug()
time.sleep(2)
plug_device.plug()
with TestRun.step(
"Verify that all cores from plugged core device are listed with "
"proper status."):
for core in cores:
if core.get_status() != CoreStatus.detached:
TestRun.fail(f"Each core should be in detached state. "
f"Actual states: {casadm.list_caches().stdout}")
with TestRun.step("Remove CAS devices from core pool."):
casadm.remove_all_detached_cores()
with TestRun.step("Verify that cores are no longer listed."):
output = casadm.list_caches().stdout
for dev in core_devs:
if dev.system_path in output:
TestRun.fail(
f"CAS device is still listed in casadm list output:\n{output}"
)
def test_cli_add_remove_custom_id(shortcut):
"""
title: Test for adding and removing a core with a custom ID - short and long command
description: |
Start a new cache and add a core to it with passing a random core ID
(from allowed pool) as an argument and then remove this core from the cache.
pass_criteria:
- The core is added to the cache with a default ID
- The core is successfully removed from the cache
"""
with TestRun.step("Prepare the devices."):
cache_disk = TestRun.disks['cache']
cache_disk.create_partitions([Size(50, Unit.MebiByte)])
cache_device = cache_disk.partitions[0]
core_device = TestRun.disks['core']
with TestRun.step("Start the cache and add the core with a random ID."):
core_id = randint(*CORE_ID_RANGE)
cache = casadm.start_cache(cache_device, shortcut=shortcut, force=True)
core = casadm.add_core(cache,
core_device,
core_id=core_id,
shortcut=shortcut)
TestRun.LOGGER.info(f"Core ID: {core_id}")
with TestRun.step("Check if the core is added to the cache."):
caches = casadm_parser.get_caches()
if len(caches[0].get_core_devices()) != 1:
TestRun.fail("One core should be present in the cache.")
if caches[0].get_core_devices()[0].path != core.path:
TestRun.fail(
"The core path should be equal to the path of the core added.")
with TestRun.step("Remove the core from the cache."):
casadm.remove_core(cache.cache_id, core.core_id, shortcut=shortcut)
with TestRun.step(
"Check if the core is successfully removed from still running cache."
):
caches = casadm_parser.get_caches()
if len(caches) != 1:
TestRun.fail(
"One cache should be still present after removing the core.")
if len(caches[0].get_core_devices()) != 0:
TestRun.fail(
"No core device should be present after removing the core.")
with TestRun.step("Stop the cache."):
casadm.stop_cache(cache_id=cache.cache_id, shortcut=shortcut)
with TestRun.step("Check if the cache has successfully stopped."):
caches = casadm_parser.get_caches()
if len(caches) != 0:
TestRun.fail(
"No cache should be present after stopping the cache.")
output = casadm.list_caches(shortcut=shortcut)
cli_messages.check_stdout_msg(output, cli_messages.no_caches_running)
def get_caches(): # This method does not return inactive or detached CAS devices
from api.cas.cache import Cache
caches_list = []
lines = casadm.list_caches(OutputFormat.csv).stdout.split('\n')
for line in lines:
args = line.split(',')
if args[0] == "cache":
current_cache = Cache(Device(args[2]))
caches_list.append(current_cache)
return caches_list
def __get_core_info(self):
output = casadm.list_caches(OutputFormat.csv, by_id_path=True)
reader = csv.DictReader(io.StringIO(output.stdout))
for row in reader:
if row['type'] == "core" and row['disk'] == self.core_device.path:
return {
"core_id": row['id'],
"core_device": row['disk'],
"status": row['status'],
"exp_obj": row['device']
}
def get_cores(cache_id: int):
from api.cas.core import Core, CoreStatus
cores_list = []
lines = casadm.list_caches(OutputFormat.csv).stdout.split('\n')
is_proper_core_line = False
for line in lines:
args = line.split(',')
if args[0] == "core" and is_proper_core_line:
core_status_str = args[3].lower()
is_valid_status = CoreStatus[core_status_str].value[0] <= 1
if is_valid_status:
cores_list.append(Core(args[2], cache_id))
if args[0] == "cache":
is_proper_core_line = True if int(args[1]) == cache_id else False
return cores_list
def test_cli_start_stop_default_value(prepare_and_cleanup, shortcut):
prepare()
cache_device = next(disk for disk in TestProperties.dut.disks
if disk.disk_type == DiskType.optane)
casadm.start_cache(cache_device, shortcut=shortcut, force=True)
caches = casadm_parser.get_caches()
assert len(caches) == 1
assert caches[0].cache_device.system_path == cache_device.system_path
casadm.stop_cache(cache_id=caches[0].cache_id, shortcut=shortcut)
output = casadm.list_caches(shortcut=shortcut)
caches = casadm_parser.get_caches()
assert len(caches) == 0
assert output.stdout == "No caches running"
def get_flushing_progress(cache_id: int, core_id: int = None):
casadm_output = casadm.list_caches(OutputFormat.csv).stdout.splitlines()
for line in casadm_output:
line_elements = line.split(',')
if core_id is not None and line_elements[0] == "core" \
and int(line_elements[1]) == core_id \
or core_id is None and line_elements[0] == "cache" \
and int(line_elements[1]) == cache_id:
try:
flush_line_elements = line_elements[3].split()
flush_percent = flush_line_elements[1][1:]
return float(flush_percent)
except Exception:
break
raise CmdException(f"There is no flushing progress in casadm list output. (cache {cache_id}"
f"{' core ' + str(core_id) if core_id is not None else ''})",
casadm_output)
def test_cli_start_stop_custom_id(shortcut):
"""
title: Test for starting a cache with a custom ID - short and long command
description: |
Start a new cache with a random ID (from allowed pool) and then stop this cache.
pass_criteria:
- The cache has successfully started with a custom ID
- The cache has successfully stopped
"""
with TestRun.step("Prepare the device for the cache."):
cache_device = TestRun.disks['cache']
cache_device.create_partitions([Size(500, Unit.MebiByte)])
cache_device = cache_device.partitions[0]
with TestRun.step("Start the cache with a random ID."):
cache_id = randint(*CACHE_ID_RANGE)
cache = casadm.start_cache(cache_device,
cache_id=cache_id,
shortcut=shortcut,
force=True)
TestRun.LOGGER.info(f"Cache ID: {cache_id}")
with TestRun.step("Check if the cache has started successfully."):
caches = casadm_parser.get_caches()
if len(caches) != 1:
TestRun.fail(
f"There is a wrong number of caches found in the OS: {len(caches)}. "
f"Should be only 1.")
if cache.cache_device.path != cache_device.path:
TestRun.fail(f"The cache has started using a wrong device:"
f" {cache.cache_device.path}."
f"\nShould use {cache_device.path}.")
with TestRun.step("Stop the cache."):
casadm.stop_cache(cache.cache_id, shortcut=shortcut)
with TestRun.step("Check if the cache has stopped properly."):
caches = casadm_parser.get_caches()
if len(caches) != 0:
TestRun.fail(
f"There is a wrong number of caches found in the OS: {len(caches)}."
f"\nNo cache should be present after stopping the cache.")
output = casadm.list_caches(shortcut=shortcut)
cli_messages.check_stdout_msg(output, cli_messages.no_caches_running)
def test_remove_inactive_devices():
"""
title: Validate removing inactive CAS devices.
description: |
Validate that it is possible to remove inactive CAS devices when there are no dirty
cache lines associated with them and that removing CAS devices is prevented otherwise
(unless ‘force’ option is used).
pass_criteria:
- No kernel error
- Removing CAS devices without dirty data is successful.
- Removing CAS devices with dirty data without ‘force’ option is blocked.
- Removing CAS devices with dirty data with ‘force’ option is successful.
"""
with TestRun.step("Prepare devices."):
devices = prepare_devices([("cache", 1), ("core", 4)])
cache_dev = devices["cache"].partitions[0]
core_devs = devices["core"].partitions
plug_device = devices["core"]
with TestRun.step("Start cache and add four cores."):
cache = casadm.start_cache(cache_dev,
cache_mode=CacheMode.WB,
force=True)
cores = []
for d in core_devs:
cores.append(cache.add_core(d))
with TestRun.step(
"Create init config file using current CAS configuration."):
InitConfig.create_init_config_from_running_configuration()
with TestRun.step("Run random writes to all CAS devices."):
run_fio([c.system_path for c in cores])
with TestRun.step(
"Flush dirty data from two CAS devices and verify than other two "
"contain dirty data."):
for core in cores:
if core.core_id % 2 == 0:
core.flush_core()
if core.get_dirty_blocks() != Size.zero():
TestRun.fail("Failed to flush CAS device.")
elif core.get_dirty_blocks() == Size.zero():
TestRun.fail("There should be dirty data on CAS device.")
with TestRun.step("Stop cache without flushing dirty data."):
cache.stop(no_data_flush=True)
with TestRun.step("Unplug core disk."):
plug_device.unplug()
with TestRun.step("Load cache."):
casadm.load_cache(cache_dev)
with TestRun.step(
"Verify that all previously created CAS devices are listed with "
"proper status."):
for core in cores:
if core.get_status() != CoreStatus.inactive:
TestRun.fail(f"Each core should be in inactive state. "
f"Actual states:\n{casadm.list_caches().stdout}")
with TestRun.step(
"Try removing CAS device without ‘force’ option. Verify that for "
"dirty CAS devices operation is blocked, proper message is displayed "
"and device is still listed."):
shuffle(cores)
for core in cores:
try:
dirty_blocks = core.get_dirty_blocks()
core.remove_core()
if dirty_blocks != Size.zero():
TestRun.fail(
"Removing dirty CAS device should be impossible but remove "
"command executed without any error.")
TestRun.LOGGER.info(
"Removing core with force option skipped for clean CAS device."
)
except CmdException as e:
if dirty_blocks == Size.zero():
TestRun.fail(
"Removing clean CAS device should be possible but remove "
"command returned an error.")
TestRun.LOGGER.info(
"Remove operation without force option is blocked for "
"dirty CAS device as expected.")
cli_messages.check_stderr_msg(
e.output, cli_messages.remove_inactive_core)
output = casadm.list_caches().stdout
if core.system_path not in output:
TestRun.fail(
f"CAS device is not listed in casadm list output but it should be."
f"\n{output}")
core.remove_core(force=True)
with TestRun.step("Plug missing disk and stop cache."):
plug_device.plug()
casadm.stop_all_caches()
def test_one_core_fail(cache_mode):
"""
title: Test if OpenCAS correctly handles failure of one of multiple core devices.
description: |
When one core device fails in a single cache instance all blocks previously occupied
should be available to other core devices.
Test is without pass through mode.
pass_criteria:
- No system crash.
- Second core is able to use OpenCAS.
"""
with TestRun.step("Prepare one cache and two core devices."):
cache_dev = TestRun.disks['cache']
cache_dev.create_partitions([Size(1, Unit.GibiByte)] * 2)
cache_part = cache_dev.partitions[0]
core_dev1 = TestRun.disks['core1'] # This device would be unplugged.
core_dev1.create_partitions([Size(2, Unit.GibiByte)])
core_part1 = core_dev1.partitions[0]
core_dev2 = TestRun.disks['core2']
core_dev2.create_partitions([Size(2, Unit.GibiByte)])
core_part2 = core_dev2.partitions[0]
Udev.disable()
with TestRun.step("Start cache"):
cache = casadm.start_cache(cache_part, cache_mode, force=True)
caches_count = len(casadm_parser.get_caches())
if caches_count != 1:
TestRun.fail(
f"Expected caches count: 1; Actual caches count: {caches_count}."
)
with TestRun.step("Add both core devices to cache."):
core1 = cache.add_core(core_part1)
core2 = cache.add_core(core_part2)
cores_count = len(casadm_parser.get_cores(cache.cache_id))
if cores_count != 2:
TestRun.fail(
f"Expected cores count: 2; Actual cores count: {cores_count}.")
with TestRun.step("Change sequential cutoff policy."):
cache.set_seq_cutoff_policy(SeqCutOffPolicy.never)
with TestRun.step("Fill cache with pages from the first core."):
dd_builder(cache_mode, core1, cache.size).run()
cache_occupied_blocks_before = cache.get_occupancy()
with TestRun.step("Unplug the first core device."):
core_dev1.unplug()
with TestRun.step("Check if core device is really out of cache."):
output = str(casadm.list_caches().stdout.splitlines())
if core_part1.path in output:
TestRun.fail("The first core device should be unplugged!")
with TestRun.step("Check if the remaining core is able to use cache."):
dd_builder(cache_mode, core2, Size(100, Unit.MebiByte)).run()
if not float(core2.get_occupancy().get_value()) > 0:
TestRun.LOGGER.error(
"The remaining core is not able to use cache.")
with TestRun.step(
"Check if occupancy from the first core is removed from cache."):
# Cache occupancy cannot be lower than before the first core fails and after that
# should be equal to the sum of occupancy of the first and the remaining core
cache_occupied_blocks_after = cache.get_occupancy()
if cache_occupied_blocks_before > cache_occupied_blocks_after \
or cache_occupied_blocks_after != core2.get_occupancy() + core1.get_occupancy():
TestRun.fail("Blocks previously occupied by the first core "
"aren't released after this core failure.")
with TestRun.step("Stop cache."):
casadm.stop_all_caches()
with TestRun.step("Plug back the first core."):
core_dev1.plug()
def test_recovery_unplug_cache_fs(cache_mode, cls, filesystem, direct):
"""
title: Test for recovery after cache drive removal - test with filesystem.
description: |
Verify that unflushed data can be safely recovered after, when SSD drive is removed
after write completion - test with filesystem.
pass_criteria:
- CAS recovers successfully after cache drive unplug
- No data corruption
"""
with TestRun.step("Prepare devices"):
cache_disk = TestRun.disks['cache']
core_disk = TestRun.disks['core']
cache_disk.create_partitions([Size(2, Unit.GibiByte)])
core_disk.create_partitions([Size(16, Unit.GibiByte)])
cache_device = cache_disk.partitions[0]
core_device = core_disk.partitions[0]
with TestRun.step("Create test files."):
source_file, target_file = create_test_files(test_file_size)
source_file_md5 = source_file.md5sum()
with TestRun.step("Create filesystem on core device."):
core_device.create_filesystem(filesystem)
with TestRun.step("Start cache and add core."):
cache = casadm.start_cache(cache_device, cache_mode, cls)
core = cache.add_core(core_device)
with TestRun.step("Mount CAS device."):
core.mount(mount_point)
with TestRun.step("Copy file to CAS."):
copy_file(source=source_file.full_path, target=test_file_path,
size=test_file_size, direct="oflag" if direct else None)
TestRun.LOGGER.info(str(core.get_statistics()))
with TestRun.step("Unmount CAS device."):
core.unmount()
with TestRun.step("Unplug cache device."):
cache_disk.unplug()
TestRun.LOGGER.info(f"List caches:\n{casadm.list_caches().stdout}")
TestRun.LOGGER.info(f"Dirty blocks on cache: "
f"{cache.get_dirty_blocks().get_value(Unit.Blocks4096)}")
with TestRun.step("Stop cache."):
try:
cache.stop()
TestRun.fail("Stopping the cache should be aborted without --no-flush flag.")
except CmdException as e:
TestRun.LOGGER.info(str(e.output))
try:
cache.stop(no_data_flush=True)
TestRun.LOGGER.warning("Expected stopping cache with errors with --no-flush flag.")
except CmdException as e1:
cli_messages.check_stderr_msg(e1.output, cli_messages.stop_cache_errors)
with TestRun.step("Plug missing cache device."):
TestRun.LOGGER.info(str(casadm.list_caches(by_id_path=False)))
cache_disk.plug()
with TestRun.step("Load cache."):
cache = casadm.load_cache(cache_device)
TestRun.LOGGER.info(f"Dirty blocks on cache: "
f"{cache.get_dirty_blocks().get_value(Unit.Blocks4096)}")
with TestRun.step("Stop cache with data flush."):
cache.stop()
with TestRun.step("Mount core device."):
core_device.mount(mount_point)
with TestRun.step("Copy file from core device and check md5sum."):
copy_file(source=test_file_path, target=target_file.full_path,
size=test_file_size, direct="iflag" if direct else None)
target_file_md5 = target_file.md5sum()
compare_files(source_file_md5, target_file_md5)
with TestRun.step("Unmount core device and remove files."):
core_device.unmount()
try:
target_file.remove()
source_file.remove()
except Exception:
# On some OSes files at /tmp location are automatically removed after DUT hard reset
pass
