async def test_rpc_timeout(container_mod, mayastor_mod, create_temp_file,
pool_file):
ms1_c = container_mod.get("ms1")
ms1 = mayastor_mod.get("ms1")
uuid = str(guid.uuid4())
timeout_pattern = 'destroy_replica: gRPC method timed out, args: DestroyReplicaRequest {{ uuid: "{}" }}'.format(
uuid)
# Create a pool and a big replica (> 1 GB)
ms1.pool_create("pool1", "uring://{}".format(pool_file))
ms1.replica_create("pool1", uuid, 2 * 1024 * 1024 * 1024)
# Set timeout to the minimum possible value and reconnect handles.
ms1.set_timeout(1)
ms1.reconnect()
# Destroy the replica and trigger the timeout.
with pytest.raises(grpc.RpcError) as error:
ms1.replica_destroy(uuid)
assert error.value.code() == grpc.StatusCode.INVALID_ARGUMENT
# Should not see error message pattern, as we expect the call to be timed out.
assert str(ms1_c.logs()).find(timeout_pattern) == -1
# Try to destroy the replica one more time - the call should complete
# without assertions.
# We expect this call to detect the incompleted previous call and emit
# a warning.
ms1.replica_destroy(uuid)
# Now we should see the evidence that the gRPC call was timed out.
assert str(ms1_c.logs()).find(timeout_pattern) > 0
def create_nexus_2(mayastor_mod, nexus_name, nexus_uuid, min_cntlid_2,
resv_key_2):
""" Create a 2nd nexus on ms0 with the same 2 replicas but with resv_key_2 """
hdls = mayastor_mod
NEXUS_NAME = nexus_name
replicas = []
list = mayastor_mod.get("ms3").nexus_list_v2()
nexus = next(n for n in list if n.name == NEXUS_NAME)
replicas.append(nexus.children[0].uri)
replicas.append(nexus.children[1].uri)
NEXUS_UUID, size_mb = nexus_uuid
hdls["ms0"].nexus_create_v2(
NEXUS_NAME,
NEXUS_UUID,
size_mb,
min_cntlid_2,
min_cntlid_2 + 9,
resv_key_2,
0,
replicas,
)
uri = hdls["ms0"].nexus_publish(NEXUS_NAME)
assert len(hdls["ms0"].bdev_list()) == 1
assert len(hdls["ms1"].bdev_list()) == 2
assert len(hdls["ms2"].bdev_list()) == 2
assert len(hdls["ms3"].bdev_list()) == 1
dev = nvme_connect(uri)
yield dev
nvme_disconnect(uri)
hdls["ms0"].nexus_destroy(NEXUS_NAME)
async def test_mayastor_features(mayastor_mod):
ms1 = mayastor_mod.get("ms1")
ms3 = mayastor_mod.get("ms3")
for replica, ms in ((True, ms1), (False, ms3)):
ms_info = ms.mayastor_info()
assert ms_info.version.startswith("v0.")
# Should see ANA disabled on mayastors where environment
# variable is not set.
features = ms_info.supportedFeatures
if replica:
assert features.asymmetricNamespaceAccess == False
else:
assert features.asymmetricNamespaceAccess == True
async def test_controller_stats(mayastor_mod, create_replicas, create_nexus):
nexus = mayastor_mod.get("ms3")
mscli = get_msclient().with_json_output().with_url(nexus.ip_address())
# Should see exactly 2 controllers on the nexus instance.
assure_controllers(mscli, create_replicas)
# Check that stats exist for all controllers and are initially empty.
output = mscli("controller", "stats")
assert len(output["controllers"]) == len(create_replicas)
names = [c["name"] for c in output["controllers"]]
for r in create_replicas:
c = ctrl_name_from_uri(r)
assert c in names, "Controller for replica %s not found" % c
for s in output["controllers"]:
stats = s["stats"]
for n in stats.keys():
assert stats[n] == 0, "Stat %s is not zero for a new controller" % n
# Issue I/O to replicas and make sure stats reflect that.
job = FioSpdk("job1", "readwrite", create_nexus, runtime=5).build()
await run_cmd_async(job)
target_stats = ["num_read_ops", "num_write_ops", "bytes_read", "bytes_written"]
cached_stats = {"num_write_ops": 0, "bytes_written": 0}
output = mscli("controller", "stats")
assert len(output["controllers"]) == 2
for c in output["controllers"]:
stats = c["stats"]
# Make sure all related I/O stats are counted.
for s in target_stats:
assert stats[s] > 0, "I/O stat %s is zero after active I/O operations" % s
# Make sure IOPS and number of bytes are sane (fio uses 4k block).
assert (
stats["bytes_read"] == stats["num_read_ops"] * 4096
), "Read IOPs don't match number of bytes"
assert (
stats["bytes_written"] == stats["num_write_ops"] * 4096
), "Write IOPs don't match number of bytes"
# Check that write-related stats are equal for all controllers in the same nexus
# and make sure all unrelated stats remained untouched.
for s in stats.keys():
if s in cached_stats:
# Cache I/O stat to check across all controllers against the same value.
if cached_stats[s] == 0:
cached_stats[s] = stats[s]
else:
# I/O stats for all controllers in a nexus must be equal.
assert cached_stats[s] == stats[s], (
"I/O statistics %s for replicas mismatch" % s
)
elif s not in target_stats:
assert stats[s] == 0, "Unrelated I/O stat %s got impacted by I/O" % s
def create_replicas(mayastor_mod):
ms1 = mayastor_mod.get("ms1")
ms2 = mayastor_mod.get("ms2")
replicas = []
for m in (ms1, ms2):
p = m.pool_create(POOL_NAME, "malloc:///disk0?size_mb=100")
assert p.state == pb.POOL_ONLINE
r = m.replica_create(POOL_NAME, str(uuid.uuid4()), 32 * 1024 * 1024)
replicas.append(r.uri)
yield replicas
try:
for m in (ms1, ms2):
m.pool_destroy(POOL_NAME)
except Exception:
pass
def create_nexuses(mayastor_mod, create_replicas):
uris = []
nvme_disconnect_all()
for n in ["ms2", "ms3"]:
ms = mayastor_mod.get(n)
ms.nexus_create(NEXUS_GUID, 32 * 1024 * 1024, create_replicas)
uri = ms.nexus_publish(NEXUS_GUID)
uris.append(uri)
yield uris
nvme_disconnect_all()
for n in ["ms2", "ms3"]:
ms = mayastor_mod.get(n)
ms.nexus_destroy(NEXUS_GUID)
async def test_controller_list(mayastor_mod, create_replicas, create_nexus):
replica1 = mayastor_mod.get("ms1")
replica2 = mayastor_mod.get("ms2")
replicas = [replica1, replica2]
nexus = mayastor_mod.get("ms3")
mscli = get_msclient().with_json_output()
# Should not see any controllers on replica instances.
for r in replicas:
output = mscli.with_url(r.ip_address())("controller", "list")
assert len(output["controllers"]) == 0
# Should see exactly 2 controllers on the nexus instance.
mscli.with_url(nexus.ip_address())
assure_controllers(mscli, create_replicas)
# Should not see a controller for the removed replica.
nexus.nexus_remove_replica(NEXUS_GUID, create_replicas[0])
assure_controllers(mscli, create_replicas[1:])
# Should see controller for the newly added replica.
nexus.nexus_add_replica(NEXUS_GUID, create_replicas[0], True)
assure_controllers(mscli, create_replicas)
async def test_io_policy(create_replicas, create_nexuses, mayastor_mod):
devs = connect_multipath_nexuses(create_nexuses)
assert devs[0] == devs[1], "Paths are different for multipath nexus"
# Make sure all we see exactly 2 paths and all paths are 'live optimized'
device = devs[0]
descr = nvme_list_subsystems(device)
paths = descr["Subsystems"][0]["Paths"]
assert len(paths) == 2, "Number of paths to Nexus mismatches"
for p in paths:
assert p["State"] == "live"
assert p["ANAState"] == "optimized"
# Make sure there are 2 virtual NVMe controllers for the namespace.
ns = os.path.basename(device)
for i in range(2):
cname = ns.replace("n1", "c%dn1" % i)
cpath = "/sys/block/%s" % cname
l = os.readlink(cpath)
assert l.startswith(
"../devices/virtual/nvme-fabrics/ctl/"
), "Path device is not a virtual controller"
# Make sure virtual NVMe namespace exists for multipath nexus.
l = os.readlink("/sys/block/%s" % ns)
assert l.startswith(
"../devices/virtual/nvme-subsystem/nvme-subsys"
), "No virtual NVMe subsystem exists for multipath Nexus"
# Make sure I/O policy is NUMA.
subsys = descr["Subsystems"][0]["Name"]
pfile = "/sys/class/nvme-subsystem/%s/iopolicy" % subsys
assert os.path.isfile(pfile), "No iopolicy file exists"
with open(pfile) as f:
iopolicy = f.read().strip()
assert iopolicy == "numa", "I/O policy is not NUMA"
# Make sure ANA state is reported properly for both nexuses.
for n in ["ms2", "ms3"]:
ms = mayastor_mod.get(n)
nexuses = ms.nexus_list_v2()
assert len(nexuses) == 1, "Number of nexuses mismatches"
assert (
nexuses[0].ana_state == pb.NVME_ANA_OPTIMIZED_STATE
), "ANA state of nexus mismatches"
def test_nexus_multipath(
create_nexus,
create_nexus_2,
nexus_name,
nexus_uuid,
mayastor_mod,
resv_key,
resv_key_2,
):
"""Create 2 nexuses, each with 2 replicas, with different NVMe reservation keys"""
uri = create_nexus
uri2 = create_nexus_2
NEXUS_UUID, _ = nexus_uuid
NEXUS_NAME = nexus_name
resv_key = resv_key
resv_key_2 = resv_key_2
list = mayastor_mod.get("ms3").nexus_list_v2()
nexus = next(n for n in list if n.name == NEXUS_NAME)
assert nexus.uuid == NEXUS_UUID
for c in range(2):
child_uri = nexus.children[c].uri
dev = nvme_connect(child_uri)
report = nvme_resv_report(dev)
assert (report["rtype"] == 5
), "should have write exclusive, all registrants reservation"
assert report["regctl"] == 2, "should have 2 registered controllers"
assert report[
"ptpls"] == 0, "should have Persist Through Power Loss State of 0"
for i in range(2):
assert (report["regctlext"][i]["cntlid"] == 0xFFFF
), "should have dynamic controller ID"
assert report["regctlext"][0]["rkey"] == resv_key
assert report["regctlext"][1]["rkey"] == resv_key_2
assert (report["regctlext"][0]["rcsts"] & 0x1) == 1
assert (report["regctlext"][1]["rcsts"] & 0x1) == 0
nvme_disconnect(child_uri)
def create_nexus_3_dev(mayastor_mod, nexus_name, nexus_uuid, replica_uuid,
min_cntlid_3, resv_key_3):
""" Create a 3rd nexus on ms1 with the same 2 replicas but with resv_key_3 """
hdls = mayastor_mod
NEXUS_NAME = nexus_name
replicas = []
list = mayastor_mod.get("ms3").nexus_list_v2()
nexus = next(n for n in list if n.name == NEXUS_NAME)
# use loopback until nvme initiator can connect to target in same instance
REP_UUID, rep_size_mb = replica_uuid
replicas.append("loopback:///" + REP_UUID)
replicas.append(nexus.children[1].uri)
NEXUS_UUID, size_mb = nexus_uuid
hdls["ms1"].nexus_create_v2(
NEXUS_NAME,
NEXUS_UUID,
size_mb,
min_cntlid_3,
min_cntlid_3 + 9,
resv_key_3,
0,
replicas,
)
uri = hdls["ms1"].nexus_publish(NEXUS_NAME)
assert len(hdls["ms0"].bdev_list()) == 1
assert len(hdls["ms1"].bdev_list()) == 3
assert len(hdls["ms2"].bdev_list()) == 2
assert len(hdls["ms3"].bdev_list()) == 1
dev = nvme_connect(uri)
yield dev
nvme_disconnect(uri)
hdls["ms1"].nexus_destroy(NEXUS_NAME)
def create_nexus(mayastor_mod, create_replicas):
ms3 = mayastor_mod.get("ms3")
ms3.nexus_create(NEXUS_GUID, 32 * 1024 * 1024, create_replicas)
uri = ms3.nexus_publish(NEXUS_GUID)
yield uri
ms3.nexus_destroy(NEXUS_GUID)