def test_statefulset_pod_deletion(core_api, storage_class,
statefulset): # NOQA
"""
Test that a StatefulSet can spin up a new Pod with the same data after a
previous Pod has been deleted.
1. Create a StatefulSet with VolumeClaimTemplate and Longhorn.
2. Wait for pods to run.
3. Write some data to one of the pod.
4. Delete that pod.
5. Wait for the StatefulSet to recreate the pod
6. Verify the data in the pod.
"""
statefulset_name = 'statefulset-pod-deletion-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
test_pod_name = statefulset_name + '-' + \
str(randrange(statefulset['spec']['replicas']))
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
write_pod_volume_data(core_api, test_pod_name, test_data)
# Not using delete_and_wait_pod here because there is the small chance the
# StatefulSet recreates the Pod quickly enough where the function won't
# detect that the Pod was deleted, which will time out and throw an error.
core_api.delete_namespaced_pod(name=test_pod_name,
namespace='default',
body=k8sclient.V1DeleteOptions())
wait_statefulset(statefulset)
resp = read_volume_data(core_api, test_pod_name)
assert resp == test_data
def test_statefulset_pod_deletion(core_api, storage_class,
statefulset): # NOQA
"""
Test that a StatefulSet can spin up a new Pod with the same data after a
previous Pod has been deleted.
This test will only work in a CSI environment. It will automatically be
disabled in FlexVolume environments.
"""
statefulset_name = 'statefulset-pod-deletion-test'
update_test_manifests(statefulset, storage_class, statefulset_name)
test_pod_name = statefulset_name + '-' + \
str(randrange(statefulset['spec']['replicas']))
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
write_volume_data(core_api, test_pod_name, test_data)
# Not using delete_and_wait_pod here because there is the small chance the
# StatefulSet recreates the Pod quickly enough where the function won't
# detect that the Pod was deleted, which will time out and throw an error.
core_api.delete_namespaced_pod(name=test_pod_name,
namespace='default',
body=k8sclient.V1DeleteOptions())
wait_statefulset(statefulset)
resp = read_volume_data(core_api, test_pod_name)
assert resp == test_data
def test_statefulset_pod_deletion(core_api, storage_class, statefulset): # NOQA
"""
Test that a StatefulSet can spin up a new Pod with the same data after a
previous Pod has been deleted.
This test will only work in a CSI environment. It will automatically be
disabled in FlexVolume environments.
"""
statefulset_name = 'statefulset-pod-deletion-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
test_pod_name = statefulset_name + '-' + \
str(randrange(statefulset['spec']['replicas']))
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
write_volume_data(core_api, test_pod_name, test_data)
# Not using delete_and_wait_pod here because there is the small chance the
# StatefulSet recreates the Pod quickly enough where the function won't
# detect that the Pod was deleted, which will time out and throw an error.
core_api.delete_namespaced_pod(name=test_pod_name, namespace='default',
body=k8sclient.V1DeleteOptions())
wait_statefulset(statefulset)
resp = read_volume_data(core_api, test_pod_name)
assert resp == test_data
def test_offline_node_with_attached_volume_and_pod(
client, core_api, volume_name, make_deployment_with_pvc,
reset_cluster_ready_status): # NOQA
"""
Test offline node with attached volume and pod
1. Create PV/PVC/Deployment manifest.
2. Update deployment's tolerations to 20 seconds to speed up test
3. Update deployment's node affinity rule to avoid the current node
4. Create volume, PV/PVC and deployment.
5. Find the pod in the deployment and write `test_data` into it
6. Shutdown the node pod is running on
7. Wait for deployment to delete the pod
1. Deployment cannot delete the pod here because kubelet doesn't
response
8. Force delete the terminating pod
9. Wait for the new pod to be created and the volume attached
10. Check `test_data` in the new pod
"""
toleration_seconds = 20
apps_api = get_apps_api_client()
cloudprovider = detect_cloudprovider()
volume_name = generate_volume_name()
pv_name = volume_name + "-pv"
pvc_name = volume_name + "-pvc"
deployment_name = volume_name + "-dep"
longhorn_test_node_name = get_self_host_id()
deployment_manifest = make_deployment_with_pvc(deployment_name, pvc_name)
unreachable_toleration = {
"key": "node.kubernetes.io/unreachable",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": toleration_seconds
}
not_ready_toleration = {
"key": "node.kubernetes.io/not-ready",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": toleration_seconds
}
deployment_manifest["spec"]["template"]["spec"]["tolerations"] =\
[unreachable_toleration, not_ready_toleration]
node_affinity_roles = {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kubernetes.io/hostname",
"operator": "NotIn",
"values": [longhorn_test_node_name]
}]
}]
}
}
}
deployment_manifest["spec"]["template"]["spec"]["affinity"] =\
node_affinity_roles
longhorn_volume = create_and_check_volume(client, volume_name, size=SIZE)
wait_for_volume_detached(client, volume_name)
create_pv_for_volume(client, core_api, longhorn_volume, pv_name)
create_pvc_for_volume(client, core_api, longhorn_volume, pvc_name)
create_and_wait_deployment(apps_api, deployment_manifest)
deployment_label_selector =\
"name=" + deployment_manifest["metadata"]["labels"]["name"]
deployment_pod_list =\
core_api.list_namespaced_pod(namespace="default",
label_selector=deployment_label_selector)
assert deployment_pod_list.items.__len__() == 1
pod_name = deployment_pod_list.items[0].metadata.name
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
node_name = deployment_pod_list.items[0].spec.node_name
node = cloudprovider.node_id(node_name)
cloudprovider.node_shutdown(node)
k8s_node_down = wait_for_node_down_k8s(node_name, core_api)
assert k8s_node_down
client = get_longhorn_api_client()
longhorn_node_down = wait_for_node_down_longhorn(node_name, client)
assert longhorn_node_down
time.sleep(toleration_seconds + 5)
for i in range(TERMINATING_POD_RETRYS):
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
terminating_pod_name = None
for pod in deployment_pod_list.items:
if pod.metadata.__getattribute__("deletion_timestamp") is not None:
terminating_pod_name = pod.metadata.name
break
if terminating_pod_name is not None:
break
else:
time.sleep(TERMINATING_POD_INTERVAL)
assert terminating_pod_name is not None
core_api.delete_namespaced_pod(namespace="default",
name=terminating_pod_name,
grace_period_seconds=0)
delete_and_wait_pod(core_api, terminating_pod_name)
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
assert deployment_pod_list.items.__len__() == 1
wait_for_volume_detached(client, volume_name)
wait_for_volume_healthy(client, volume_name)
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
assert deployment_pod_list.items.__len__() == 1
new_pod_name = deployment_pod_list.items[0].metadata.name
wait_pod(new_pod_name)
resp_data = read_volume_data(core_api, new_pod_name)
assert test_data == resp_data