def wait_new_replica_ready(client, volume_name, replica_names): # NOQA
"""
Wait for a new replica to be found on the specified volume. Trigger a
failed assertion if one can't be found.
:param client: The Longhorn client to use in the request.
:param volume_name: The name of the volume.
:param replica_names: The list of names of the volume's old replicas.
"""
new_replica_ready = False
wait_for_rebuild_complete(client, volume_name)
for _ in range(RETRY_COUNTS):
v = client.by_id_volume(volume_name)
for r in v.replicas:
if r["name"] not in replica_names and r["running"] and \
r["mode"] == "RW":
new_replica_ready = True
break
if new_replica_ready:
break
sleep(RETRY_INTERVAL)
assert new_replica_ready
def test_allow_volume_creation_with_degraded_availability_csi(
client, core_api, apps_api, make_deployment_with_pvc): # NOQA
"""
Test Allow Volume Creation with Degraded Availability (CSI)
Requirement:
1. Set `allow-volume-creation-with-degraded-availability` to true.
2. Set `node-level-soft-anti-affinity` to false.
Steps:
1. Disable scheduling for node 3.
2. Create a Deployment Pod with a volume and 3 replicas.
1. After the volume is attached, scheduling error should be seen.
3. Write data to the Pod.
4. Scale down the deployment to 0 to detach the volume.
1. Scheduled condition should become true.
5. Scale up the deployment back to 1 and verify the data.
1. Scheduled condition should become false.
6. Enable the scheduling for node 3.
1. Volume should start rebuilding on the node 3 soon.
2. Once the rebuilding starts, the scheduled condition should become
true.
7. Once rebuild finished, scale down and back the deployment to verify
the data.
"""
setting = client.by_id_setting(common.SETTING_DEGRADED_AVAILABILITY)
client.update(setting, value="true")
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
nodes = client.list_node()
node3 = nodes[2]
client.update(node3, allowScheduling=False)
vol = common.create_and_check_volume(client, generate_volume_name(),
size=str(500 * Mi))
pv_name = vol.name + "-pv"
common.create_pv_for_volume(client, core_api, vol, pv_name)
pvc_name = vol.name + "-pvc"
common.create_pvc_for_volume(client, core_api, vol, pvc_name)
deployment_name = vol.name + "-dep"
deployment = make_deployment_with_pvc(deployment_name, pvc_name)
deployment["spec"]["replicas"] = 3
apps_api.create_namespaced_deployment(body=deployment, namespace='default')
common.wait_for_volume_status(client, vol.name,
common.VOLUME_FIELD_STATE,
common.VOLUME_STATE_ATTACHED)
common.wait_scheduling_failure(client, vol.name)
data_path = "/data/test"
pod = common.wait_and_get_any_deployment_pod(core_api, deployment_name)
common.write_pod_volume_random_data(core_api, pod.metadata.name,
data_path, common.DATA_SIZE_IN_MB_2)
created_md5sum = get_pod_data_md5sum(core_api, pod.metadata.name,
data_path)
deployment['spec']['replicas'] = 0
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment_name)
vol = common.wait_for_volume_detached(client, vol.name)
assert vol.conditions[VOLUME_CONDITION_SCHEDULED]['status'] == "True"
deployment['spec']['replicas'] = 1
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment_name)
common.wait_for_volume_status(client, vol.name,
common.VOLUME_FIELD_STATE,
common.VOLUME_STATE_ATTACHED)
common.wait_for_volume_condition_scheduled(client, vol.name, "status",
common.CONDITION_STATUS_FALSE)
pod = common.wait_and_get_any_deployment_pod(core_api, deployment_name)
assert created_md5sum == get_pod_data_md5sum(core_api,
pod.metadata.name,
data_path)
client.update(node3, allowScheduling=True)
common.wait_for_rebuild_start(client, vol.name)
vol = client.by_id_volume(vol.name)
assert vol.conditions[VOLUME_CONDITION_SCHEDULED]['status'] == "True"
common.wait_for_rebuild_complete(client, vol.name)
deployment['spec']['replicas'] = 0
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment_name)
common.wait_for_volume_detached(client, vol.name)
deployment['spec']['replicas'] = 1
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment_name)
common.wait_for_volume_status(client, vol.name,
common.VOLUME_FIELD_STATE,
common.VOLUME_STATE_ATTACHED)
pod = common.wait_and_get_any_deployment_pod(core_api, deployment_name)
assert created_md5sum == get_pod_data_md5sum(core_api,
pod.metadata.name,
data_path)
def test_data_locality_basic(client, core_api, volume_name, pod,
settings_reset): # NOQA
"""
Test data locality basic feature
Context:
Data Locality feature allows users to have an option to keep a local
replica on the same node as the consuming pod.
Longhorn is currently supporting 2 modes:
- disabled: Longhorn does not try to keep a local replica
- best-effort: Longhorn try to keep a local replica
See manual tests at:
https://github.com/longhorn/longhorn/issues/1045#issuecomment-680706283
Steps:
Case 1: Test that Longhorn builds a local replica on the engine node
1. Create a volume(1) with 1 replica and dataLocality set to disabled
2. Find node where the replica is located on.
Let's call the node is replica-node
3. Attach the volume to a node different than replica-node.
Let call the node is engine-node
4. Write 200MB data to volume(1)
5. Use a retry loop to verify that Longhorn does not create
a replica on the engine-node
6. Update dataLocality to best-effort for volume(1)
7. Use a retry loop to verify that Longhorn creates and rebuilds
a replica on the engine-node and remove the other replica
8. detach the volume(1) and attach it to a different node.
Let's call the new node is new-engine-node and the old
node is old-engine-node
9. Wait for volume(1) to finish attaching
10. Use a retry loop to verify that Longhorn creates and rebuilds
a replica on the new-engine-node and remove the replica on
old-engine-node
Case 2: Test that Longhorn prioritizes deleting replicas on the same node
1. Add the tag AVAIL to node-1 and node-2
2. Set node soft anti-affinity to `true`.
3. Create a volume(2) with 3 replicas and dataLocality set to best-effort
4. Use a retry loop to verify that all 3 replicas are on node-1 and
node-2, no replica is on node-3
5. Attach volume(2) to node-3
6. User a retry loop to verify that there is no replica on node-3 and
we can still read/write to volume(2)
7. Find the node which contains 2 replicas.
Let call the node is most-replica-node
8. Set the replica count to 2 for volume(2)
9. Verify that Longhorn remove one replica from most-replica-node
Case 3: Test that the volume is not corrupted if there is an unexpected
detachment during building local replica
1. Remove the tag AVAIL from node-1 and node-2
Set node soft anti-affinity to `false`.
2. Create a volume(3) with 1 replicas and dataLocality set to best-effort
3. Attach volume(3) to node-3.
4. Use a retry loop to verify that volume(3) has only 1 replica on node-3
5. Write 800MB data to volume(3)
6. Detach volume(3)
7. Attach volume(3) to node-1
8. Use a retry loop to:
Wait until volume(3) finishes attaching.
Wait until Longhorn start rebuilding a replica on node-1
Immediately detach volume(3)
9. Verify that the replica on node-1 is in ERR state.
10. Attach volume(3) to node-1
11. Wait until volume(3) finishes attaching.
12. Use a retry loop to verify the Longhorn cleanup the ERR replica,
rebuild a new replica on node-1, and remove the replica on node-3
Case 4: Make sure failed to schedule local replica doesn't block the
the creation of other replicas.
1. Disable scheduling for node-3
2. Create a vol with 1 replica, `dataLocality = best-effort`.
The replica is scheduled on a node (say node-1)
3. Attach vol to node-3. There is a fail-to-schedule
replica with Spec.HardNodeAffinity=node-3
4. Increase numberOfReplica to 3. Verify that the replica set contains:
one on node-1, one on node-2, one failed replica
with Spec.HardNodeAffinity=node-3.
5. Decrease numberOfReplica to 2. Verify that the replica set contains:
one on node-1, one on node-2, one failed replica
with Spec.HardNodeAffinity=node-3.
6. Decrease numberOfReplica to 1. Verify that the replica set contains:
one on node-1 or node-2, one failed replica
with Spec.HardNodeAffinity=node-3.
7. Decrease numberOfReplica to 2. Verify that the replica set contains:
one on node-1, one on node-2, one failed replica
with Spec.HardNodeAffinity=node-3.
8. Turn off data locality by set `dataLocality=disabled` for the vol.
Verify that the replica set contains: one on node-1, one on node-2
9. clean up
"""
# Case 1: Test that Longhorn builds a local replica on the engine node
nodes = client.list_node()
default_data_locality_setting = \
client.by_id_setting(SETTING_DEFAULT_DATA_LOCALITY)
try:
client.update(default_data_locality_setting, value="disabled")
except Exception as e:
print("Exception when update Default Data Locality setting",
default_data_locality_setting, e)
volume1_name = volume_name + "-1"
volume1_size = str(500 * Mi)
volume1_data_path = "/data/test"
pv1_name = volume1_name + "-pv"
pvc1_name = volume1_name + "-pvc"
pod1_name = volume1_name + "-pod"
pod1 = pod
pod1['metadata']['name'] = pod1_name
volume1 = create_and_check_volume(client,
volume1_name,
num_of_replicas=1,
size=volume1_size)
volume1 = client.by_id_volume(volume1_name)
create_pv_for_volume(client, core_api, volume1, pv1_name)
create_pvc_for_volume(client, core_api, volume1, pvc1_name)
volume1 = client.by_id_volume(volume1_name)
volume1_replica_node = volume1.replicas[0]['hostId']
volume1_attached_node = None
for node in nodes:
if node.name != volume1_replica_node:
volume1_attached_node = node.name
break
assert volume1_attached_node is not None
pod1['spec']['volumes'] = [{
"name": "pod-data",
"persistentVolumeClaim": {
"claimName": pvc1_name
}
}]
pod1['spec']['nodeSelector'] = \
{"kubernetes.io/hostname": volume1_attached_node}
create_and_wait_pod(core_api, pod1)
write_pod_volume_random_data(core_api, pod1_name, volume1_data_path,
DATA_SIZE_IN_MB_2)
for i in range(10):
volume1 = client.by_id_volume(volume1_name)
assert len(volume1.replicas) == 1
assert volume1.replicas[0]['hostId'] != volume1_attached_node
time.sleep(1)
volume1 = client.by_id_volume(volume1_name)
volume1.updateDataLocality(dataLocality="best-effort")
for _ in range(RETRY_COUNTS):
volume1 = client.by_id_volume(volume1_name)
assert volume1[VOLUME_FIELD_ROBUSTNESS] == VOLUME_ROBUSTNESS_HEALTHY
if len(volume1.replicas) == 1 and \
volume1.replicas[0]['hostId'] == volume1_attached_node:
break
time.sleep(RETRY_INTERVAL)
assert len(volume1.replicas) == 1
assert volume1.replicas[0]['hostId'] == volume1_attached_node
delete_and_wait_pod(core_api, pod1_name)
volume1 = wait_for_volume_detached(client, volume1_name)
volume1_replica_node = volume1.replicas[0]['hostId']
volume1_attached_node = None
for node in nodes:
if node.name != volume1_replica_node:
volume1_attached_node = node.name
break
assert volume1_attached_node is not None
pod1['spec']['nodeSelector'] = \
{"kubernetes.io/hostname": volume1_attached_node}
create_and_wait_pod(core_api, pod1)
for _ in range(RETRY_COUNTS):
volume1 = client.by_id_volume(volume1_name)
assert volume1[VOLUME_FIELD_ROBUSTNESS] == VOLUME_ROBUSTNESS_HEALTHY
if len(volume1.replicas) == 1 and \
volume1.replicas[0]['hostId'] == volume1_attached_node:
break
time.sleep(RETRY_INTERVAL)
assert len(volume1.replicas) == 1
assert volume1.replicas[0]['hostId'] == volume1_attached_node
delete_and_wait_pod(core_api, pod1_name)
wait_for_volume_detached(client, volume1_name)
# Case 2: Test that Longhorn prioritizes deleting replicas on the same node
node1 = nodes[0]
node2 = nodes[1]
node3 = nodes[2]
client.update(node1, allowScheduling=True, tags=["AVAIL"])
client.update(node2, allowScheduling=True, tags=["AVAIL"])
replica_node_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
try:
client.update(replica_node_soft_anti_affinity_setting, value="true")
except Exception as e:
print(
"Exception when update "
"Replica Node Level Soft Anti-Affinity setting",
replica_node_soft_anti_affinity_setting, e)
volume2_name = volume_name + "-2"
volume2_size = str(500 * Mi)
pv2_name = volume2_name + "-pv"
pvc2_name = volume2_name + "-pvc"
pod2_name = volume2_name + "-pod"
pod2 = pod
pod2['metadata']['name'] = pod2_name
volume2 = client.create_volume(name=volume2_name,
size=volume2_size,
numberOfReplicas=3,
nodeSelector=["AVAIL"],
dataLocality="best-effort")
volume2 = wait_for_volume_detached(client, volume2_name)
volume2 = client.by_id_volume(volume2_name)
create_pv_for_volume(client, core_api, volume2, pv2_name)
create_pvc_for_volume(client, core_api, volume2, pvc2_name)
volume2 = client.by_id_volume(volume2_name)
pod2['spec']['volumes'] = [{
"name": "pod-data",
"persistentVolumeClaim": {
"claimName": pvc2_name
}
}]
pod2['spec']['nodeSelector'] = {"kubernetes.io/hostname": node3.name}
create_and_wait_pod(core_api, pod2)
volume2 = wait_for_volume_healthy(client, volume2_name)
for replica in volume2.replicas:
assert replica["hostId"] != node3.name
volume2.updateReplicaCount(replicaCount=2)
# 2 Healthy replicas and 1 replica failed to schedule
# The failed to schedule replica is the local replica on node3
volume2 = wait_for_volume_replica_count(client, volume2_name, 3)
volume2 = client.by_id_volume(volume2_name)
volume2_healthy_replicas = []
for replica in volume2.replicas:
if replica.running is True:
volume2_healthy_replicas.append(replica)
assert len(volume2_healthy_replicas) == 2
volume2_rep1 = volume2_healthy_replicas[0]
volume2_rep2 = volume2_healthy_replicas[1]
assert volume2_rep1["hostId"] != volume2_rep2["hostId"]
delete_and_wait_pod(core_api, pod2_name)
wait_for_volume_detached(client, volume2_name)
# Case 3: Test that the volume is not corrupted if there is an unexpected
# detachment during building local replica
client.update(node1, allowScheduling=True, tags=[])
client.update(node2, allowScheduling=True, tags=[])
replica_node_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
try:
client.update(replica_node_soft_anti_affinity_setting, value="false")
except Exception as e:
print(
"Exception when update "
"Replica Node Level Soft Anti-Affinity setting",
replica_node_soft_anti_affinity_setting, e)
volume3_name = volume_name + "-3"
volume3_size = str(1 * Gi)
volume3_data_path = "/data/test"
pv3_name = volume3_name + "-pv"
pvc3_name = volume3_name + "-pvc"
pod3_name = volume3_name + "-pod"
pod3 = pod
pod3['metadata']['name'] = pod3_name
volume3 = client.create_volume(name=volume3_name,
size=volume3_size,
numberOfReplicas=1)
volume3 = wait_for_volume_detached(client, volume3_name)
volume3 = client.by_id_volume(volume3_name)
create_pv_for_volume(client, core_api, volume3, pv3_name)
create_pvc_for_volume(client, core_api, volume3, pvc3_name)
volume3 = client.by_id_volume(volume3_name)
pod3['spec']['volumes'] = [{
"name": "pod-data",
"persistentVolumeClaim": {
"claimName": pvc3_name
}
}]
pod3['spec']['nodeSelector'] = {"kubernetes.io/hostname": node3.name}
create_and_wait_pod(core_api, pod3)
volume3 = wait_for_volume_healthy(client, volume3_name)
write_pod_volume_random_data(core_api, pod3_name, volume3_data_path,
DATA_SIZE_IN_MB_4)
volume3.updateDataLocality(dataLocality="best-effort")
volume3 = client.by_id_volume(volume3_name)
if volume3.replicas[0]['hostId'] != node3.name:
wait_for_rebuild_start(client, volume3_name)
volume3 = client.by_id_volume(volume3_name)
assert len(volume3.replicas) == 2
wait_for_rebuild_complete(client, volume3_name)
volume3 = wait_for_volume_replica_count(client, volume3_name, 1)
assert volume3.replicas[0]["hostId"] == node3.name
delete_and_wait_pod(core_api, pod3_name)
pod3['spec']['nodeSelector'] = {"kubernetes.io/hostname": node1.name}
create_and_wait_pod(core_api, pod3)
wait_for_rebuild_start(client, volume3_name)
crash_engine_process_with_sigkill(client, core_api, volume3_name)
delete_and_wait_pod(core_api, pod3_name)
wait_for_volume_detached(client, volume3_name)
volume3 = client.by_id_volume(volume3_name)
assert len(volume3.replicas) == 1
assert volume3.replicas[0]["hostId"] == node3.name
create_and_wait_pod(core_api, pod3)
wait_for_rebuild_start(client, volume3_name)
volume3 = client.by_id_volume(volume3_name)
assert len(volume3.replicas) == 2
wait_for_rebuild_complete(client, volume3_name)
# Wait for deletion of extra replica
volume3 = wait_for_volume_replica_count(client, volume3_name, 1)
assert volume3.replicas[0]["hostId"] == node1.name
assert volume3.replicas[0]["mode"] == "RW"
assert volume3.replicas[0]["running"] is True
delete_and_wait_pod(core_api, pod3_name)
wait_for_volume_detached(client, volume3_name)
# Case 4: Make sure failed to schedule local replica doesn't block the
# the creation of other replicas.
replica_node_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
try:
client.update(replica_node_soft_anti_affinity_setting, value="false")
except Exception as e:
print(
"Exception when update "
"Replica Node Level Soft Anti-Affinity setting",
replica_node_soft_anti_affinity_setting, e)
client.update(node3, allowScheduling=False)
volume4_name = volume_name + "-4"
volume4_size = str(1 * Gi)
volume4 = client.create_volume(name=volume4_name,
size=volume4_size,
numberOfReplicas=1,
dataLocality="best-effort")
volume4 = wait_for_volume_detached(client, volume4_name)
volume4 = client.by_id_volume(volume4_name)
volume4_replica_name = volume4.replicas[0]["name"]
volume4.attach(hostId=node3.name)
wait_for_volume_healthy(client, volume4_name)
volume4 = client.by_id_volume(volume4_name)
assert len(volume4.replicas) == 2
for replica in volume4.replicas:
if replica["name"] == volume4_replica_name:
assert replica["running"] is True
assert replica["mode"] == "RW"
else:
assert replica["running"] is False
assert replica["mode"] == ""
assert volume4.conditions.scheduled.reason == \
"LocalReplicaSchedulingFailure"
volume4 = volume4.updateReplicaCount(replicaCount=3)
volume4 = wait_for_volume_degraded(client, volume4_name)
v4_node1_replica_count = 0
v4_node2_replica_count = 0
v4_failed_replica_count = 0
for replica in volume4.replicas:
if replica["hostId"] == node1.name:
v4_node1_replica_count += 1
elif replica["hostId"] == node2.name:
v4_node2_replica_count += 1
elif replica["hostId"] == "":
v4_failed_replica_count += 1
assert v4_node1_replica_count == 1
assert v4_node2_replica_count == 1
assert v4_failed_replica_count > 0
volume4 = volume4.updateReplicaCount(replicaCount=2)
volume4 = wait_for_volume_replica_count(client, volume4_name, 3)
v4_node1_replica_count = 0
v4_node2_replica_count = 0
v4_failed_replica_count = 0
for replica in volume4.replicas:
if replica["hostId"] == node1.name:
v4_node1_replica_count += 1
elif replica["hostId"] == node2.name:
v4_node2_replica_count += 1
elif replica["hostId"] == "":
v4_failed_replica_count += 1
assert v4_node1_replica_count == 1
assert v4_node2_replica_count == 1
assert v4_failed_replica_count > 0
volume4 = volume4.updateReplicaCount(replicaCount=1)
volume4 = wait_for_volume_replica_count(client, volume4_name, 2)
v4_node1_replica_count = 0
v4_node2_replica_count = 0
v4_failed_replica_count = 0
for replica in volume4.replicas:
if replica["hostId"] == node1.name:
v4_node1_replica_count += 1
elif replica["hostId"] == node2.name:
v4_node2_replica_count += 1
elif replica["hostId"] == "":
v4_failed_replica_count += 1
assert v4_node1_replica_count + v4_node2_replica_count == 1
assert v4_failed_replica_count == 1
volume4 = volume4.updateDataLocality(dataLocality="disabled")
volume4 = volume4.updateReplicaCount(replicaCount=2)
running_replica_count = 0
for _ in range(RETRY_COUNTS):
volume4 = client.by_id_volume(volume4_name)
running_replica_count = 0
for r in volume4.replicas:
if r.failedAt == "" and r.running is True:
running_replica_count += 1
if running_replica_count == 2:
break
time.sleep(RETRY_INTERVAL)
assert running_replica_count == 2
v4_node1_replica_count = 0
v4_node2_replica_count = 0
v4_node3_replica_count = 0
for replica in volume4.replicas:
wait_for_replica_running(client, volume4_name, replica["name"])
if replica["hostId"] == node1.name:
v4_node1_replica_count += 1
elif replica["hostId"] == node2.name:
v4_node2_replica_count += 1
elif replica["hostId"] == node3.name:
v4_node3_replica_count += 1
assert v4_node1_replica_count == 1
assert v4_node2_replica_count == 1
assert v4_node3_replica_count == 0