def volume_iscsi_basic_test(clients, volume_name, base_image=""): # NOQA
# get a random client
for host_id, client in clients.iteritems():
break
volume = create_and_check_volume(client, volume_name, 3, SIZE, base_image,
"iscsi")
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 1
assert volumes[0]["name"] == volume["name"]
assert volumes[0]["size"] == volume["size"]
assert volumes[0]["numberOfReplicas"] == volume["numberOfReplicas"]
assert volumes[0]["state"] == volume["state"]
assert volumes[0]["created"] == volume["created"]
assert volumes[0]["frontend"] == "iscsi"
endpoint = get_volume_endpoint(volumes[0])
assert endpoint.startswith("iscsi://")
try:
dev = iscsi_login(endpoint)
volume_rw_test(dev)
finally:
iscsi_logout(endpoint)
cleanup_volume(client, volume)
def test_volume_update_replica_count(clients, volume_name): # NOQA
for host_id, client in clients.iteritems():
break
replica_count = 3
volume = create_and_check_volume(client, volume_name, replica_count)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
replica_count = 5
volume = volume.updateReplicaCount(replicaCount=replica_count)
volume = common.wait_for_volume_degraded(client, volume_name)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == replica_count
old_replica_count = replica_count
replica_count = 2
volume = volume.updateReplicaCount(replicaCount=replica_count)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == old_replica_count
volume.replicaRemove(name=volume["replicas"][0]["name"])
volume.replicaRemove(name=volume["replicas"][1]["name"])
volume.replicaRemove(name=volume["replicas"][2]["name"])
volume = common.wait_for_volume_replica_count(client, volume_name,
replica_count)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == replica_count
client.delete(volume)
wait_for_volume_delete(client, volume_name)
def test_soft_anti_affinity_scheduling(client, volume_name): # NOQA
"""
Test that volumes with Soft Anti-Affinity work as expected.
With Soft Anti-Affinity, a new replica should still be scheduled on a node
with an existing replica, which will result in "Healthy" state but limited
redundancy.
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_SOFT_ANTI_AFFINITY)
client.update(setting, value="true")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
replica_names = map(lambda replica: replica.name, volume["replicas"])
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica["name"])
wait_new_replica_ready(client, volume_name, replica_names)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_soft_anti_affinity_detach(client, volume_name): # NOQA
"""
Test that volumes with Soft Anti-Affinity can detach and reattach to a
node properly.
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_SOFT_ANTI_AFFINITY)
client.update(setting, value="true")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
replica_names = map(lambda replica: replica.name, volume["replicas"])
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica["name"])
wait_new_replica_ready(client, volume_name, replica_names)
volume = wait_for_volume_healthy(client, volume_name)
volume.detach()
volume = wait_for_volume_detached(client, volume_name)
assert len(volume["replicas"]) == 3
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_hard_anti_affinity_offline_rebuild(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity can build new replicas during
the attaching process once a valid node is available.
Once a new replica has been built as part of the attaching process, the
volume should be Healthy again.
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
replica_names = map(lambda replica: replica.name, volume["replicas"])
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica["name"])
volume = wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
volume.detach()
volume = wait_for_volume_detached(client, volume_name)
client.update(node, allowScheduling=True)
volume.attach(hostId=host_id)
wait_new_replica_ready(client, volume_name, replica_names)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_hard_anti_affinity_live_rebuild(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity can build new replicas live once
a valid node is available.
If no nodes without existing replicas are available, the volume should
remain in "Degraded" state. However, once one is available, the replica
should now be scheduled successfully, with the volume returning to
"Healthy" state.
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
replica_names = map(lambda replica: replica.name, volume["replicas"])
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica["name"])
wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
# Allow scheduling on host node again
client.update(node, allowScheduling=True)
wait_new_replica_ready(client, volume_name, replica_names)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_hard_anti_affinity_detach(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity are still able to detach and
reattach to a node properly, even in degraded state.
1. Create a volume and attach to the current node
2. Generate and write `data` to the volume.
3. Set `soft anti-affinity` to false
4. Disable current node's scheduling.
5. Remove the replica on the current node
1. Verify volume will be in degraded state.
2. Verify volume reports condition `scheduled == false`
6. Detach the volume.
7. Verify that volume only have 2 replicas
1. Unhealthy replica will be removed upon detach.
8. Attach the volume again.
1. Verify volume will be in degraded state.
2. Verify volume reports condition `scheduled == false`
3. Verify only two of three replicas of volume are healthy.
4. Verify the remaining replica doesn't have `replica.HostID`, meaning
it's unscheduled
9. Check volume `data`
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica.name)
volume = wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
volume.detach()
volume = wait_for_volume_detached(client, volume_name)
assert len(volume.replicas) == 2
volume.attach(hostId=host_id)
# Make sure we're still not getting another successful replica.
volume = wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
assert sum([
1 for replica in volume.replicas
if replica.running and replica.mode == "RW"
]) == 2
assert sum([1 for replica in volume.replicas if not replica.hostId]) == 1
assert len(volume.replicas) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def create_volume_with_replica_on_host(client, volume_name): # NOQA
lht_hostId = get_self_host_id()
nodes = client.list_node()
volume = create_and_check_volume(client, volume_name, len(nodes), SIZE)
volume.attach(hostId=lht_hostId, disableFrontend=False)
wait_for_volume_healthy(client, volume_name)
return volume
def test_hard_anti_affinity_scheduling(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity work as expected.
With Hard Anti-Affinity, scheduling on nodes with existing replicas should
be forbidden, resulting in "Degraded" state.
1. Create a volume and attach to the current node
2. Generate and write `data` to the volume.
3. Set `soft anti-affinity` to false
4. Disable current node's scheduling.
5. Remove the replica on the current node
1. Verify volume will be in degraded state.
2. Verify volume reports condition `scheduled == false`
3. Verify only two of three replicas of volume are healthy.
4. Verify the remaining replica doesn't have `replica.HostID`, meaning
it's unscheduled
6. Check volume `data`
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica.name)
# Instead of waiting for timeout and lengthening the tests a significant
# amount we can make sure the scheduling isn't working by making sure the
# volume becomes Degraded and reports a scheduling error.
wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
# While there are three replicas that should exist to meet the Volume's
# request, only two of those volumes should actually be Healthy.
volume = client.by_id_volume(volume_name)
assert sum([
1 for replica in volume.replicas
if replica.running and replica.mode == "RW"
]) == 2
# Confirm that the final volume is an unscheduled volume.
assert sum([1 for replica in volume.replicas if not replica.hostId]) == 1
# Three replicas in total should still exist.
assert len(volume.replicas) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_attach_without_frontend(clients, volume_name): # NOQA
for host_id, client in clients.iteritems():
break
volume = create_and_check_volume(client, volume_name)
lht_hostId = get_self_host_id()
volume.attach(hostId=lht_hostId, disableFrontend=False)
common.wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume["disableFrontend"] is False
assert volume["frontend"] == "blockdev"
snap1_data = write_volume_random_data(volume)
snap1 = volume.snapshotCreate()
write_volume_random_data(volume)
volume.snapshotCreate()
volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=lht_hostId, disableFrontend=True)
common.wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
engine = get_volume_engine(volume)
assert volume["disableFrontend"] is True
assert volume["frontend"] == "blockdev"
assert engine["endpoint"] == ""
volume.snapshotRevert(name=snap1["name"])
volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=lht_hostId, disableFrontend=False)
common.wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume["disableFrontend"] is False
assert volume["frontend"] == "blockdev"
check_volume_data(volume, snap1_data)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
def test_hard_anti_affinity_offline_rebuild(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity can build new replicas during
the attaching process once a valid node is available.
Once a new replica has been built as part of the attaching process, the
volume should be Healthy again.
1. Create a volume and attach to the current node
2. Generate and write `data` to the volume.
3. Set `soft anti-affinity` to false
4. Disable current node's scheduling.
5. Remove the replica on the current node
1. Verify volume will be in degraded state.
2. Verify volume reports condition `scheduled == false`
6. Detach the volume.
7. Enable current node's scheduling.
8. Attach the volume again.
9. Wait for volume to become healthy with 3 replicas
10. Check volume `data`
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
replica_names = map(lambda replica: replica.name, volume.replicas)
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica.name)
volume = wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
volume.detach()
volume = wait_for_volume_detached(client, volume_name)
client.update(node, allowScheduling=True)
volume.attach(hostId=host_id)
wait_new_replica_ready(client, volume_name, replica_names)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_hard_anti_affinity_live_rebuild(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity can build new replicas live once
a valid node is available.
If no nodes without existing replicas are available, the volume should
remain in "Degraded" state. However, once one is available, the replica
should now be scheduled successfully, with the volume returning to
"Healthy" state.
1. Create a volume and attach to the current node
2. Generate and write `data` to the volume.
3. Set `soft anti-affinity` to false
4. Disable current node's scheduling.
5. Remove the replica on the current node
1. Verify volume will be in degraded state.
2. Verify volume reports condition `scheduled == false`
6. Enable the current node's scheduling
7. Wait for volume to start rebuilding and become healthy again
8. Check volume `data`
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
replica_names = map(lambda replica: replica.name, volume.replicas)
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica.name)
wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
# Allow scheduling on host node again
client.update(node, allowScheduling=True)
wait_new_replica_ready(client, volume_name, replica_names)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_soft_anti_affinity_detach(client, volume_name): # NOQA
"""
Test that volumes with Soft Anti-Affinity can detach and reattach to a
node properly.
1. Create a volume and attach to the current node.
2. Generate and write `data` to the volume
3. Set `soft anti-affinity` to true
4. Disable current node's scheduling.
5. Remove the replica on the current node
6. Wait for the new replica to be rebuilt
7. Detach the volume.
8. Verify there are 3 replicas
9. Attach the volume again. Verify there are still 3 replicas
10. Verify the `data`.
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="true")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
replica_names = list(map(lambda replica: replica.name, volume.replicas))
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica.name)
wait_new_replica_ready(client, volume_name, replica_names)
volume = wait_for_volume_healthy(client, volume_name)
volume.detach()
volume = wait_for_volume_detached(client, volume_name)
assert len(volume.replicas) == 3
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_hard_anti_affinity_scheduling(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity work as expected.
With Hard Anti-Affinity, scheduling on nodes with existing replicas should
be forbidden, resulting in "Degraded" state.
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica["name"])
# Instead of waiting for timeout and lengthening the tests a significant
# amount we can make sure the scheduling isn't working by making sure the
# volume becomes Degraded and reports a scheduling error.
volume = wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
# While there are three replicas that should exist to meet the Volume's
# request, only two of those volumes should actually be Healthy.
assert sum([
1 for replica in volume["replicas"]
if replica["running"] and replica["mode"] == "RW"
]) == 2
# Confirm that the final volume is an unscheduled volume.
assert sum([1 for replica in volume["replicas"]
if not replica["hostId"]]) == 1
# Three replicas in total should still exist.
assert len(volume["replicas"]) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_hard_anti_affinity_detach(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity are still able to detach and
reattach to a node properly, even in degraded state.
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica["name"])
volume = wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
volume.detach()
volume = wait_for_volume_detached(client, volume_name)
assert len(volume["replicas"]) == 2
volume.attach(hostId=host_id)
# Make sure we're still not getting another successful replica.
volume = wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
assert sum([
1 for replica in volume["replicas"]
if replica["running"] and replica["mode"] == "RW"
]) == 2
assert sum([1 for replica in volume["replicas"]
if not replica["hostId"]]) == 1
assert len(volume["replicas"]) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_soft_anti_affinity_scheduling(client, volume_name): # NOQA
"""
Test that volumes with Soft Anti-Affinity work as expected.
With Soft Anti-Affinity, a new replica should still be scheduled on a node
with an existing replica, which will result in "Healthy" state but limited
redundancy.
1. Create a volume and attach to the current node
2. Generate and write `data` to the volume.
3. Set `soft anti-affinity` to true
4. Disable current node's scheduling.
5. Remove the replica on the current node
6. Wait for the volume to complete rebuild. Volume should have 3 replicas.
7. Verify `data`
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="true")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
replica_names = list(map(lambda replica: replica.name, volume.replicas))
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica.name)
wait_new_replica_ready(client, volume_name, replica_names)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)