def delete_replica(client, volume_name):
volume = client.by_id_volume(volume_name)
replica_count = len(volume.replicas)
healthy_replica_count = 0
for replica in volume.replicas:
if replica.running is True and replica.mode == "RW":
healthy_replica_count += 1
# return if there is no or only one healthy replica left
if healthy_replica_count == 1:
print("skipped, only one healthy replica found", end=" ")
return
if healthy_replica_count == 0:
print("skipped, no healthy replicas found", end=" ")
return
replica_id = randrange(0, replica_count)
replica_name = volume["replicas"][replica_id]["name"]
volume.replicaRemove(name=replica_name)
wait_for_volume_degraded(client, volume_name)
global WAIT_REPLICA_REBUILD
if WAIT_REPLICA_REBUILD is None:
WAIT_REPLICA_REBUILD = bool(random.getrandbits(1))
if WAIT_REPLICA_REBUILD is True:
wait_for_volume_replica_count(client, volume_name, replica_count)
replica_names = map(lambda replica: replica.name, volume["replicas"])
wait_new_replica_ready(client, volume_name, replica_names)
def test_zone_tags(client, core_api, volume_name, k8s_node_zone_tags): # NOQA
"""
Test anti affinity zone feature
1. Add Kubernetes zone labels to the nodes
1. Only two zones now: zone1 and zone2
2. Create a volume with two replicas
3. Verify zone1 and zone2 either has one replica.
4. Remove a random replica and wait for volume to finish rebuild
5. Verify zone1 and zone2 either has one replica.
6. Repeat step 4-5 a few times.
7. Update volume to 3 replicas, make sure they're scheduled on 3 nodes
8. Remove a random replica and wait for volume to finish rebuild
9. Make sure replicas are on different nodes
10. Repeat step 8-9 a few times
"""
wait_longhorn_node_zone_updated(client)
volume = create_and_check_volume(client, volume_name, num_of_replicas=2)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
zone1_replica_count = get_zone_replica_count(client, volume_name, ZONE1)
zone2_replica_count = get_zone_replica_count(client, volume_name, ZONE2)
assert zone1_replica_count == zone2_replica_count
for i in range(randrange(3, 5)):
volume = client.by_id_volume(volume_name)
replica_count = len(volume.replicas)
assert replica_count == 2
replica_id = randrange(0, replica_count)
replica_name = volume.replicas[replica_id].name
volume.replicaRemove(name=replica_name)
wait_for_volume_degraded(client, volume_name)
wait_for_volume_healthy(client, volume_name)
wait_for_volume_replica_count(client, volume_name, replica_count)
volume = client.by_id_volume(volume_name)
replica_names = map(lambda replica: replica.name, volume["replicas"])
wait_new_replica_ready(client, volume_name, replica_names)
zone1_replica_count = \
get_zone_replica_count(client, volume_name, ZONE1)
zone2_replica_count = \
get_zone_replica_count(client, volume_name, ZONE2)
assert zone1_replica_count == zone2_replica_count
volume.updateReplicaCount(replicaCount=3)
wait_for_volume_degraded(client, volume_name)
wait_for_volume_replica_count(client, volume_name, 3)
wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
lh_node_names = list(map(lambda node: node.name, client.list_node()))
for replica in volume.replicas:
lh_node_names.remove(replica.hostId)
assert lh_node_names == []
for i in range(randrange(3, 5)):
volume = client.by_id_volume(volume_name)
replica_count = len(volume.replicas)
assert replica_count == 3
replica_id = randrange(0, replica_count)
replica_name = volume.replicas[replica_id].name
volume.replicaRemove(name=replica_name)
wait_for_volume_degraded(client, volume_name)
wait_for_volume_healthy(client, volume_name)
wait_for_volume_replica_count(client, volume_name, replica_count)
volume = client.by_id_volume(volume_name)
lh_node_names = list(map(lambda node: node.name, client.list_node()))
for replica in volume.replicas:
lh_node_names.remove(replica.hostId)
assert lh_node_names == []