Esempio n. 1
0
async def test_nexus_2_remote_mirror_kill_all_fio(containers, mayastors,
                                                  create_nexus, nexus_uuid):
    """Create a nexus on ms3 with replicas on ms1 and ms2. Start fio_spdk for
    15s. Kill ms2 after 4s, ms1 after 4s. Assume the fail with a
    ChildProcessError is due to fio bailing out. Remove the nexus from ms3.
    """

    uri = create_nexus
    NEXUS_UUID, _ = nexus_uuid

    job = FioSpdk("job1", "randwrite", uri).build()

    try:
        # create an event loop polling the async processes for completion
        await asyncio.gather(
            run_cmd_async(job),
            kill_after(containers.get("ms2"), 4),
            kill_after(containers.get("ms1"), 4),
        )
    except ChildProcessError:
        pass
    except Exception as e:
        # if it's not a child process error fail the test
        raise (e)
    finally:
        list = mayastors.get("ms3").nexus_list()
        nexus = next(n for n in list if n.uuid == NEXUS_UUID)

        assert nexus.state == pb.NEXUS_FAULTED

        assert nexus.children[0].state == pb.CHILD_FAULTED
        assert nexus.children[1].state == pb.CHILD_FAULTED
Esempio n. 2
0
async def test_nexus_2_remote_mirror_kill_1(containers, mayastors,
                                            create_nexus, nexus_uuid):
    """Create a nexus on ms3 with replicas on ms1 and ms2. Sleep for 10s. Kill
    ms2 after 4s, verify that the second child is degraded.
    """

    uri = create_nexus
    NEXUS_UUID, _ = nexus_uuid

    job = "sleep 10"

    try:
        # create an event loop polling the async processes for completion
        await asyncio.gather(
            run_cmd_async(job),
            kill_after(containers.get("ms2"), 4),
        )
    except Exception as e:
        raise (e)
    finally:
        list = mayastors.get("ms3").nexus_list()
        nexus = next(n for n in list if n.uuid == NEXUS_UUID)

        assert nexus.state == pb.NEXUS_DEGRADED

        assert nexus.children[0].state == pb.CHILD_ONLINE
        assert nexus.children[1].state == pb.CHILD_FAULTED
Esempio n. 3
0
async def test_multiple_fs(containers, connect_devices, mount_devices):
    # we're now writing to files not raw devices
    files = [f"/mnt{dev}/file.dat" for dev in connect_devices]
    fio_cmd = Fio(
        f"job-fs",
        "randwrite",
        files,
        optstr="--verify=crc32 --verify_fatal=1 --verify_async=2 --size=50mb",
    ).build()
    print(fio_cmd)

    to_kill = containers.get("ms3")
    await asyncio.gather(run_cmd_async(fio_cmd), kill_after(to_kill, 3))
Esempio n. 4
0
async def test_nexus_2_mirror_kill_one(containers, mayastors, create_nexus):

    uri = create_nexus
    nvme_discover(uri)
    dev = nvme_connect(uri)
    try:
        job = Fio("job1", "rw", dev).build()
        print(job)

        to_kill = containers.get("ms2")
        await asyncio.gather(run_cmd_async(job), kill_after(to_kill, 5))

    finally:
        # disconnect target before we shutdown
        nvme_disconnect(uri)
Esempio n. 5
0
async def test_nexus_2_remote_mirror_kill_one(containers, mayastors,
                                              nexus_uuid, create_nexus):
    """
    This test does the following steps:

        - creates mayastor instances
        - creates pools on mayastor 1 and 2
        - creates replicas on those pools
        - creates a nexus on mayastor 3
        - starts fio on a remote VM (vixos1) for 15 secondsj
        - kills mayastor 2 after 4 seconds
        - assume the test to succeed
        - disconnect the VM from mayastor 3 when FIO completes
        - removes the nexus from mayastor 3
        - removes the replicas but as mayastor 2 is down, will swallow errors
        - removes the pool

    The bulk of this is done by reusing fixtures those fitures are not as
    generic as one might like at this point so look/determine if you need them
    to begin with.

    By yielding from fixtures, after the tests the function is resumed where
    yield is called.
    """

    uri = create_nexus
    dev = nvme_connect(uri)
    try:
        job = Fio("job1", "randwrite", dev).build()
        print(job)

        to_kill = containers.get("ms2")

        # create an event loop polling the async processes for completion
        await asyncio.gather(run_cmd_async(job), kill_after(to_kill, 4))

        list = mayastors.get("ms3").nexus_list()

        NEXUS_UUID, size_mb = nexus_uuid
        nexus = next(n for n in list if n.uuid == NEXUS_UUID)

        assert nexus.state == pb.NEXUS_DEGRADED
        assert nexus.children[1].state == pb.CHILD_FAULTED

    finally:
        # disconnect target before we shutdown
        nvme_disconnect(uri)
Esempio n. 6
0
async def test_nexus_2_remote_mirror_kill_one_spdk(containers, mayastors,
                                                   nexus_uuid, create_nexus):
    """
    Identical to the previous test except fio uses the SPDK ioengine
    """

    uri = create_nexus

    job = FioSpdk("job1", "randwrite", uri).build()
    print(job)

    to_kill = containers.get("ms2")
    await asyncio.gather(run_cmd_async(job), kill_after(to_kill, 4))

    list = mayastors.get("ms3").nexus_list()

    NEXUS_UUID, _ = nexus_uuid
    nexus = next(n for n in list if n.uuid == NEXUS_UUID)

    assert nexus.state == pb.NEXUS_DEGRADED
    assert nexus.children[1].state == pb.CHILD_FAULTED
Esempio n. 7
0
def test_restart(containers, mayastors, create_replicas_on_all_nodes, times):
    """
    Test that when we create replicas and destroy them the count is as expected
    At this point we have 3 nodes each with NEXUS_COUNT replicas.
    """

    node = containers.get("ms1")
    ms1 = mayastors.get("ms1")

    # kill one of the nodes, restart it, and verify we still have NEXUS_COUNT replicas
    node.kill()
    node.start()

    # must reconnect grpc
    ms1.reconnect()

    # create does import here if found
    ms1.pool_create("ms1", "aio:///tmp/ms1.img")

    # check the list has the required number of replicas
    replicas = ms1.replica_list().replicas
    assert len(replicas) == NEXUS_COUNT

    # destroy a few
    for i in range(DESTROY_COUNT):
        ms1.replica_destroy(replicas[i].uuid)

    # kill (again) and reconnect
    node.kill()
    node.start()
    ms1.reconnect()

    # verify we have correct number of replicas remaining
    ms1.pool_create("ms1", "aio:///tmp/ms1.img")
    replicas = ms1.replica_list().replicas

    assert len(replicas) + DESTROY_COUNT == NEXUS_COUNT
Esempio n. 8
0
async def test_multiple_spdk(containers, create_nexuses):
    fio_cmd = FioSpdk(f"job-spdk", "randwrite", create_nexuses).build()

    to_kill = containers.get("ms3")
    await asyncio.gather(run_cmd_async(fio_cmd), kill_after(to_kill, 3))
Esempio n. 9
0
async def test_multiple_raw(containers, connect_devices):
    fio_cmd = Fio(f"job-raw", "randwrite", connect_devices).build()
    print(fio_cmd)

    to_kill = containers.get("ms3")
    await asyncio.gather(run_cmd_async(fio_cmd), kill_after(to_kill, 3))