def connect_multipath_nexuses(uris):
dev1 = nvme_connect(uris[0])
dev2 = None
try:
dev2 = nvme_connect(uris[1])
except Exception:
# The first connect is allowed to fail due to controller ID collision.
pass
if dev2 is None:
dev2 = nvme_connect(uris[1])
return (dev1, dev2)
def create_nexus_2(mayastor_mod, nexus_name, nexus_uuid, min_cntlid_2,
resv_key_2):
""" Create a 2nd nexus on ms0 with the same 2 replicas but with resv_key_2 """
hdls = mayastor_mod
NEXUS_NAME = nexus_name
replicas = []
list = mayastor_mod.get("ms3").nexus_list_v2()
nexus = next(n for n in list if n.name == NEXUS_NAME)
replicas.append(nexus.children[0].uri)
replicas.append(nexus.children[1].uri)
NEXUS_UUID, size_mb = nexus_uuid
hdls["ms0"].nexus_create_v2(
NEXUS_NAME,
NEXUS_UUID,
size_mb,
min_cntlid_2,
min_cntlid_2 + 9,
resv_key_2,
0,
replicas,
)
uri = hdls["ms0"].nexus_publish(NEXUS_NAME)
assert len(hdls["ms0"].bdev_list()) == 1
assert len(hdls["ms1"].bdev_list()) == 2
assert len(hdls["ms2"].bdev_list()) == 2
assert len(hdls["ms3"].bdev_list()) == 1
dev = nvme_connect(uri)
yield dev
nvme_disconnect(uri)
hdls["ms0"].nexus_destroy(NEXUS_NAME)
def create_nexus(mayastor_mod, nexus_name, nexus_uuid, create_replica,
min_cntlid, resv_key):
""" Create a nexus on ms3 with 2 replicas """
hdls = mayastor_mod
replicas = create_replica
replicas = [k.uri for k in replicas]
NEXUS_UUID, size_mb = nexus_uuid
NEXUS_NAME = nexus_name
hdls["ms3"].nexus_create_v2(
NEXUS_NAME,
NEXUS_UUID,
size_mb,
min_cntlid,
min_cntlid + 9,
resv_key,
0,
replicas,
)
uri = hdls["ms3"].nexus_publish(NEXUS_NAME)
assert len(hdls["ms1"].bdev_list()) == 2
assert len(hdls["ms2"].bdev_list()) == 2
assert len(hdls["ms3"].bdev_list()) == 1
assert len(hdls["ms1"].pool_list().pools) == 1
assert len(hdls["ms2"].pool_list().pools) == 1
dev = nvme_connect(uri)
yield dev
nvme_disconnect(uri)
hdls["ms3"].nexus_destroy(NEXUS_NAME)
def test_nexus_resv_key(create_nexus_v2, nexus_name, nexus_uuid, mayastors,
resv_key):
"""Test create_nexus_v2 replica NVMe reservation key"""
uri = create_nexus_v2
NEXUS_UUID, _ = nexus_uuid
list = mayastors.get("ms3").nexus_list_v2()
nexus = next(n for n in list if n.name == nexus_name)
assert nexus.uuid == NEXUS_UUID
child_uri = nexus.children[0].uri
dev = nvme_connect(child_uri)
try:
report = nvme_resv_report(dev)
print(report)
assert (report["rtype"] == 5
), "should have write exclusive, all registrants reservation"
assert report["regctl"] == 1, "should have 1 registered controller"
assert report[
"ptpls"] == 0, "should have Persist Through Power Loss State of 0"
assert (report["regctlext"][0]["cntlid"] == 0xFFFF
), "should have dynamic controller ID"
# reservation status reserved
assert (report["regctlext"][0]["rcsts"] & 0x1) == 1
assert report["regctlext"][0]["rkey"] == resv_key
finally:
nvme_disconnect(child_uri)
async def test_namespace_guid(create_replicas, create_nexuses, mayastor_mod):
uri = create_nexuses[0]
device = nvme_connect(uri)
ns = identify_namespace(device)
nvme_disconnect(uri)
# Namespace's GUID must match Nexus GUID.
assert uuid.UUID(ns["nguid"]) == uuid.UUID(
NEXUS_GUID
), "Namespace NGID doesn't match Nexus GUID"
# Extended Unique Identifier must be zero.
assert ns["eui64"] == "0000000000000000", "Namespace EUI64 is not zero"
async def test_nexus_2_mirror_kill_one(containers, mayastors, create_nexus):
uri = create_nexus
nvme_discover(uri)
dev = nvme_connect(uri)
try:
job = Fio("job1", "rw", dev).build()
print(job)
to_kill = containers.get("ms2")
await asyncio.gather(run_cmd_async(job), kill_after(to_kill, 5))
finally:
# disconnect target before we shutdown
nvme_disconnect(uri)
async def test_nexus_2_remote_mirror_kill_one(containers, mayastors,
nexus_uuid, create_nexus):
"""
This test does the following steps:
- creates mayastor instances
- creates pools on mayastor 1 and 2
- creates replicas on those pools
- creates a nexus on mayastor 3
- starts fio on a remote VM (vixos1) for 15 secondsj
- kills mayastor 2 after 4 seconds
- assume the test to succeed
- disconnect the VM from mayastor 3 when FIO completes
- removes the nexus from mayastor 3
- removes the replicas but as mayastor 2 is down, will swallow errors
- removes the pool
The bulk of this is done by reusing fixtures those fitures are not as
generic as one might like at this point so look/determine if you need them
to begin with.
By yielding from fixtures, after the tests the function is resumed where
yield is called.
"""
uri = create_nexus
dev = nvme_connect(uri)
try:
job = Fio("job1", "randwrite", dev).build()
print(job)
to_kill = containers.get("ms2")
# create an event loop polling the async processes for completion
await asyncio.gather(run_cmd_async(job), kill_after(to_kill, 4))
list = mayastors.get("ms3").nexus_list()
NEXUS_UUID, size_mb = nexus_uuid
nexus = next(n for n in list if n.uuid == NEXUS_UUID)
assert nexus.state == pb.NEXUS_DEGRADED
assert nexus.children[1].state == pb.CHILD_FAULTED
finally:
# disconnect target before we shutdown
nvme_disconnect(uri)
async def test_nexus_cntlid(create_nexus_v2, min_cntlid):
"""Test create_nexus_v2 NVMe controller ID"""
uri = create_nexus_v2
dev = nvme_connect(uri)
try:
id_ctrl = nvme_id_ctrl(dev)
assert id_ctrl["cntlid"] == min_cntlid
# Test optional command support
oncs = id_ctrl["oncs"]
assert oncs & 0x04, "should support Dataset Management"
assert oncs & 0x08, "should support Write Zeroes"
finally:
# disconnect target before we shut down
nvme_disconnect(uri)
def connect_to_node_2(publish_to_node_2):
device = nvme_connect(publish_to_node_2)
desc = nvme_list_subsystems(device)
subsystem = desc["Subsystems"][0]
assert len(
subsystem["Paths"]) == 2, "Second nexus must be added to I/O path"
good_path_checked = False
broken_path_checked = False
for p in subsystem["Paths"]:
if p["Name"] in device:
assert p[
"State"] == "connecting", "Degraded I/O path has incorrect state"
broken_path_checked = True
else:
assert p["State"] == "live", "Healthy I/O path has incorrect state"
good_path_checked = True
assert good_path_checked, "No state reported for healthy I/O path"
assert broken_path_checked, "No state reported for broken I/O path"
def test_nexus_multipath(
create_nexus,
create_nexus_2,
nexus_name,
nexus_uuid,
mayastor_mod,
resv_key,
resv_key_2,
):
"""Create 2 nexuses, each with 2 replicas, with different NVMe reservation keys"""
uri = create_nexus
uri2 = create_nexus_2
NEXUS_UUID, _ = nexus_uuid
NEXUS_NAME = nexus_name
resv_key = resv_key
resv_key_2 = resv_key_2
list = mayastor_mod.get("ms3").nexus_list_v2()
nexus = next(n for n in list if n.name == NEXUS_NAME)
assert nexus.uuid == NEXUS_UUID
for c in range(2):
child_uri = nexus.children[c].uri
dev = nvme_connect(child_uri)
report = nvme_resv_report(dev)
assert (report["rtype"] == 5
), "should have write exclusive, all registrants reservation"
assert report["regctl"] == 2, "should have 2 registered controllers"
assert report[
"ptpls"] == 0, "should have Persist Through Power Loss State of 0"
for i in range(2):
assert (report["regctlext"][i]["cntlid"] == 0xFFFF
), "should have dynamic controller ID"
assert report["regctlext"][0]["rkey"] == resv_key
assert report["regctlext"][1]["rkey"] == resv_key_2
assert (report["regctlext"][0]["rcsts"] & 0x1) == 1
assert (report["regctlext"][1]["rcsts"] & 0x1) == 0
nvme_disconnect(child_uri)
def create_nexus_3_dev(mayastor_mod, nexus_name, nexus_uuid, replica_uuid,
min_cntlid_3, resv_key_3):
""" Create a 3rd nexus on ms1 with the same 2 replicas but with resv_key_3 """
hdls = mayastor_mod
NEXUS_NAME = nexus_name
replicas = []
list = mayastor_mod.get("ms3").nexus_list_v2()
nexus = next(n for n in list if n.name == NEXUS_NAME)
# use loopback until nvme initiator can connect to target in same instance
REP_UUID, rep_size_mb = replica_uuid
replicas.append("loopback:///" + REP_UUID)
replicas.append(nexus.children[1].uri)
NEXUS_UUID, size_mb = nexus_uuid
hdls["ms1"].nexus_create_v2(
NEXUS_NAME,
NEXUS_UUID,
size_mb,
min_cntlid_3,
min_cntlid_3 + 9,
resv_key_3,
0,
replicas,
)
uri = hdls["ms1"].nexus_publish(NEXUS_NAME)
assert len(hdls["ms0"].bdev_list()) == 1
assert len(hdls["ms1"].bdev_list()) == 3
assert len(hdls["ms2"].bdev_list()) == 2
assert len(hdls["ms3"].bdev_list()) == 1
dev = nvme_connect(uri)
yield dev
nvme_disconnect(uri)
hdls["ms1"].nexus_destroy(NEXUS_NAME)
def connect_devices(publish_nexuses):
yield [nvme_connect(nexus) for nexus in publish_nexuses]
for nexus in publish_nexuses:
nvme_disconnect(nexus)
def connect_devices(create_nexuses):
"Connect an nvmf device to each nexus."
yield [nvme_connect(nexus) for nexus in create_nexuses]
for nexus in create_nexuses:
nvme_disconnect(nexus)
def connect_to_first_path(background):
volume = background
device_uri = volume.state["target"]["deviceUri"]
yield nvme_connect(device_uri)
nvme_disconnect(device_uri)
def test_nexus_preempt_key(
create_nexus_v2,
create_nexus_2_v2,
nexus_name,
nexus_uuid,
mayastors,
resv_key_2,
):
"""Create a nexus on ms3 and ms0, with the latter preempting the NVMe
reservation key registered by ms3, verify that ms3 is no longer registered.
Verify that writes succeed via the nexus on ms0 but not ms3."""
NEXUS_UUID, _ = nexus_uuid
list = mayastors.get("ms3").nexus_list_v2()
nexus = next(n for n in list if n.name == nexus_name)
assert nexus.uuid == NEXUS_UUID
child_uri = nexus.children[0].uri
assert nexus.state == pb.NEXUS_ONLINE
assert nexus.children[0].state == pb.CHILD_ONLINE
assert nexus.children[1].state == pb.CHILD_ONLINE
dev = nvme_connect(child_uri)
try:
report = nvme_resv_report(dev)
print(report)
assert (report["rtype"] == 5
), "should have write exclusive, all registrants reservation"
assert report["regctl"] == 1, "should have 1 registered controller"
assert report[
"ptpls"] == 0, "should have Persist Through Power Loss State of 0"
assert (report["regctlext"][0]["cntlid"] == 0xFFFF
), "should have dynamic controller ID"
# reservation status reserved
assert (report["regctlext"][0]["rcsts"] & 0x1) == 1
assert report["regctlext"][0]["rkey"] == resv_key_2
finally:
nvme_disconnect(child_uri)
# verify write with nexus on ms0
uri = create_nexus_2_v2
dev = nvme_connect(uri)
job = "sudo dd if=/dev/urandom of={0} bs=512 count=1".format(dev)
try:
run_cmd(job)
finally:
nvme_disconnect(uri)
list = mayastors.get("ms0").nexus_list_v2()
nexus = next(n for n in list if n.name == nexus_name)
assert nexus.state == pb.NEXUS_ONLINE
assert nexus.children[0].state == pb.CHILD_ONLINE
assert nexus.children[1].state == pb.CHILD_ONLINE
# verify write error with nexus on ms3
uri = create_nexus_v2
dev = nvme_connect(uri)
job = "sudo dd if=/dev/urandom of={0} bs=512 count=1".format(dev)
try:
run_cmd(job)
finally:
nvme_disconnect(uri)
list = mayastors.get("ms3").nexus_list_v2()
nexus = next(n for n in list if n.name == nexus_name)
assert nexus.state == pb.NEXUS_FAULTED
assert nexus.children[0].state == pb.CHILD_FAULTED
assert nexus.children[1].state == pb.CHILD_FAULTED
def connect_nexus_2(create_nexus_2_no_destroy):
uri = create_nexus_2_no_destroy
dev = nvme_connect(uri)
return dev