async def test_nexus_2_remote_mirror_kill_all_fio(containers, mayastors,
create_nexus, nexus_uuid):
"""Create a nexus on ms3 with replicas on ms1 and ms2. Start fio_spdk for
15s. Kill ms2 after 4s, ms1 after 4s. Assume the fail with a
ChildProcessError is due to fio bailing out. Remove the nexus from ms3.
"""
uri = create_nexus
NEXUS_UUID, _ = nexus_uuid
job = FioSpdk("job1", "randwrite", uri).build()
try:
# create an event loop polling the async processes for completion
await asyncio.gather(
run_cmd_async(job),
kill_after(containers.get("ms2"), 4),
kill_after(containers.get("ms1"), 4),
)
except ChildProcessError:
pass
except Exception as e:
# if it's not a child process error fail the test
raise (e)
finally:
list = mayastors.get("ms3").nexus_list()
nexus = next(n for n in list if n.uuid == NEXUS_UUID)
assert nexus.state == pb.NEXUS_FAULTED
assert nexus.children[0].state == pb.CHILD_FAULTED
assert nexus.children[1].state == pb.CHILD_FAULTED
async def test_nexus_2_remote_mirror_kill_1(containers, mayastors,
create_nexus, nexus_uuid):
"""Create a nexus on ms3 with replicas on ms1 and ms2. Sleep for 10s. Kill
ms2 after 4s, verify that the second child is degraded.
"""
uri = create_nexus
NEXUS_UUID, _ = nexus_uuid
job = "sleep 10"
try:
# create an event loop polling the async processes for completion
await asyncio.gather(
run_cmd_async(job),
kill_after(containers.get("ms2"), 4),
)
except Exception as e:
raise (e)
finally:
list = mayastors.get("ms3").nexus_list()
nexus = next(n for n in list if n.uuid == NEXUS_UUID)
assert nexus.state == pb.NEXUS_DEGRADED
assert nexus.children[0].state == pb.CHILD_ONLINE
assert nexus.children[1].state == pb.CHILD_FAULTED
async def test_multiple_fs(containers, connect_devices, mount_devices):
# we're now writing to files not raw devices
files = [f"/mnt{dev}/file.dat" for dev in connect_devices]
fio_cmd = Fio(
f"job-fs",
"randwrite",
files,
optstr="--verify=crc32 --verify_fatal=1 --verify_async=2 --size=50mb",
).build()
print(fio_cmd)
to_kill = containers.get("ms3")
await asyncio.gather(run_cmd_async(fio_cmd), kill_after(to_kill, 3))
async def test_nexus_2_mirror_kill_one(containers, mayastors, create_nexus):
uri = create_nexus
nvme_discover(uri)
dev = nvme_connect(uri)
try:
job = Fio("job1", "rw", dev).build()
print(job)
to_kill = containers.get("ms2")
await asyncio.gather(run_cmd_async(job), kill_after(to_kill, 5))
finally:
# disconnect target before we shutdown
nvme_disconnect(uri)
async def test_nexus_2_remote_mirror_kill_one(containers, mayastors,
nexus_uuid, create_nexus):
"""
This test does the following steps:
- creates mayastor instances
- creates pools on mayastor 1 and 2
- creates replicas on those pools
- creates a nexus on mayastor 3
- starts fio on a remote VM (vixos1) for 15 secondsj
- kills mayastor 2 after 4 seconds
- assume the test to succeed
- disconnect the VM from mayastor 3 when FIO completes
- removes the nexus from mayastor 3
- removes the replicas but as mayastor 2 is down, will swallow errors
- removes the pool
The bulk of this is done by reusing fixtures those fitures are not as
generic as one might like at this point so look/determine if you need them
to begin with.
By yielding from fixtures, after the tests the function is resumed where
yield is called.
"""
uri = create_nexus
dev = nvme_connect(uri)
try:
job = Fio("job1", "randwrite", dev).build()
print(job)
to_kill = containers.get("ms2")
# create an event loop polling the async processes for completion
await asyncio.gather(run_cmd_async(job), kill_after(to_kill, 4))
list = mayastors.get("ms3").nexus_list()
NEXUS_UUID, size_mb = nexus_uuid
nexus = next(n for n in list if n.uuid == NEXUS_UUID)
assert nexus.state == pb.NEXUS_DEGRADED
assert nexus.children[1].state == pb.CHILD_FAULTED
finally:
# disconnect target before we shutdown
nvme_disconnect(uri)
async def test_nexus_2_remote_mirror_kill_one_spdk(containers, mayastors,
nexus_uuid, create_nexus):
"""
Identical to the previous test except fio uses the SPDK ioengine
"""
uri = create_nexus
job = FioSpdk("job1", "randwrite", uri).build()
print(job)
to_kill = containers.get("ms2")
await asyncio.gather(run_cmd_async(job), kill_after(to_kill, 4))
list = mayastors.get("ms3").nexus_list()
NEXUS_UUID, _ = nexus_uuid
nexus = next(n for n in list if n.uuid == NEXUS_UUID)
assert nexus.state == pb.NEXUS_DEGRADED
assert nexus.children[1].state == pb.CHILD_FAULTED
def test_restart(containers, mayastors, create_replicas_on_all_nodes, times):
"""
Test that when we create replicas and destroy them the count is as expected
At this point we have 3 nodes each with NEXUS_COUNT replicas.
"""
node = containers.get("ms1")
ms1 = mayastors.get("ms1")
# kill one of the nodes, restart it, and verify we still have NEXUS_COUNT replicas
node.kill()
node.start()
# must reconnect grpc
ms1.reconnect()
# create does import here if found
ms1.pool_create("ms1", "aio:///tmp/ms1.img")
# check the list has the required number of replicas
replicas = ms1.replica_list().replicas
assert len(replicas) == NEXUS_COUNT
# destroy a few
for i in range(DESTROY_COUNT):
ms1.replica_destroy(replicas[i].uuid)
# kill (again) and reconnect
node.kill()
node.start()
ms1.reconnect()
# verify we have correct number of replicas remaining
ms1.pool_create("ms1", "aio:///tmp/ms1.img")
replicas = ms1.replica_list().replicas
assert len(replicas) + DESTROY_COUNT == NEXUS_COUNT
async def test_multiple_spdk(containers, create_nexuses):
fio_cmd = FioSpdk(f"job-spdk", "randwrite", create_nexuses).build()
to_kill = containers.get("ms3")
await asyncio.gather(run_cmd_async(fio_cmd), kill_after(to_kill, 3))
async def test_multiple_raw(containers, connect_devices):
fio_cmd = Fio(f"job-raw", "randwrite", connect_devices).build()
print(fio_cmd)
to_kill = containers.get("ms3")
await asyncio.gather(run_cmd_async(fio_cmd), kill_after(to_kill, 3))