def delete_and_wait_statefulset_only(api, ss):
pod_data = get_statefulset_pod_info(api, ss)
apps_api = get_apps_api_client()
try:
apps_api.delete_namespaced_stateful_set(
name=ss['metadata']['name'],
namespace='default', body=k8sclient.V1DeleteOptions())
except ApiException as e:
assert e.status == 404
for i in range(RETRY_COUNTS):
ret = apps_api.list_namespaced_stateful_set(namespace='default')
found = False
for item in ret.items:
if item.metadata.name == ss['metadata']['name']:
found = True
break
if not found:
break
time.sleep(RETRY_INTERVAL)
assert not found
for p in pod_data:
wait_delete_pod(api, p['pod_name'])
def delete_and_wait_statefulset_only(api, ss):
pod_data = get_statefulset_pod_info(api, ss)
apps_api = get_apps_api_client()
try:
apps_api.delete_namespaced_stateful_set(
name=ss['metadata']['name'],
namespace='default',
body=k8sclient.V1DeleteOptions())
except ApiException as e:
assert e.status == 404
for i in range(RETRY_COUNTS):
ret = apps_api.list_namespaced_stateful_set(namespace='default')
found = False
for item in ret.items:
if item.metadata.name == ss['metadata']['name']:
found = True
break
if not found:
break
time.sleep(RETRY_INTERVAL)
assert not found
for p in pod_data:
wait_delete_pod(api, p['pod_uid'])
def wait_statefulset(statefulset_manifest):
api = get_apps_api_client()
replicas = statefulset_manifest['spec']['replicas']
for i in range(DEFAULT_POD_TIMEOUT):
s_set = api.read_namespaced_stateful_set(
name=statefulset_manifest['metadata']['name'], namespace='default')
if s_set.status.ready_replicas == replicas:
break
time.sleep(DEFAULT_POD_INTERVAL)
assert s_set.status.ready_replicas == replicas
def create_and_wait_statefulset(statefulset_manifest):
"""
Create a new StatefulSet for testing.
This function will block until all replicas in the StatefulSet are online
or it times out, whichever occurs first.
"""
api = get_apps_api_client()
api.create_namespaced_stateful_set(body=statefulset_manifest,
namespace='default')
wait_statefulset(statefulset_manifest)
def test_setting_toleration():
"""
Test toleration setting
1. Set `taint-toleration` to "key1=value1:NoSchedule; key2:InvalidEffect".
2. Verify the request fails.
3. Create a volume and attach it.
4. Set `taint-toleration` to "key1=value1:NoSchedule; key2:NoExecute".
5. Verify that cannot update toleration setting when any volume is
attached.
6. Generate and write `data1` into the volume.
7. Detach the volume.
8. Set `taint-toleration` to "key1=value1:NoSchedule; key2:NoExecute".
9. Wait for all the Longhorn system components to restart with new
toleration.
10. Verify that UI, manager, and drive deployer don't restart and
don't have new toleration.
11. Attach the volume again and verify the volume `data1`.
12. Generate and write `data2` to the volume.
13. Detach the volume.
14. Clean the `toleration` setting.
15. Wait for all the Longhorn system components to restart with no
toleration.
16. Attach the volume and validate `data2`.
17. Generate and write `data3` to the volume.
"""
client = get_longhorn_api_client() # NOQA
apps_api = get_apps_api_client() # NOQA
core_api = get_core_api_client() # NOQA
count = len(client.list_node())
setting = client.by_id_setting(SETTING_TAINT_TOLERATION)
with pytest.raises(Exception) as e:
client.update(setting,
value="key1=value1:NoSchedule; key2:InvalidEffect")
assert 'invalid effect' in str(e.value)
volume_name = "test-toleration-vol" # NOQA
volume = create_and_check_volume(client, volume_name)
volume.attach(hostId=get_self_host_id())
volume = wait_for_volume_healthy(client, volume_name)
setting_value_str = "key1=value1:NoSchedule; key2:NoExecute"
setting_value_dicts = [
{
"key": "key1",
"value": "value1",
"operator": "Equal",
"effect": "NoSchedule"
},
{
"key": "key2",
"value": None,
"operator": "Exists",
"effect": "NoExecute"
},
]
with pytest.raises(Exception) as e:
client.update(setting, value=setting_value_str)
assert 'cannot modify toleration setting before all volumes are detached' \
in str(e.value)
data1 = write_volume_random_data(volume)
check_volume_data(volume, data1)
volume.detach(hostId="")
wait_for_volume_detached(client, volume_name)
setting = client.update(setting, value=setting_value_str)
assert setting.value == setting_value_str
wait_for_toleration_update(core_api, apps_api, count, setting_value_dicts)
client, node = wait_for_longhorn_node_ready()
volume = client.by_id_volume(volume_name)
volume.attach(hostId=node)
volume = wait_for_volume_healthy(client, volume_name)
check_volume_data(volume, data1)
data2 = write_volume_random_data(volume)
check_volume_data(volume, data2)
volume.detach(hostId="")
wait_for_volume_detached(client, volume_name)
# cleanup
setting_value_str = ""
setting_value_dicts = []
setting = client.by_id_setting(SETTING_TAINT_TOLERATION)
setting = client.update(setting, value=setting_value_str)
assert setting.value == setting_value_str
wait_for_toleration_update(core_api, apps_api, count, setting_value_dicts)
client, node = wait_for_longhorn_node_ready()
volume = client.by_id_volume(volume_name)
volume.attach(hostId=node)
volume = wait_for_volume_healthy(client, volume_name)
check_volume_data(volume, data2)
data3 = write_volume_random_data(volume)
check_volume_data(volume, data3)
cleanup_volume(client, volume)
def test_statefulset_scaling(client, core_api, storage_class,
statefulset): # NOQA
"""
Test that scaling up a StatefulSet successfully provisions new volumes.
1. Create a StatefulSet with VolumeClaimTemplate and Longhorn.
2. Wait for pods to run.
3. Verify the properities of volumes.
4. Scale the StatefulSet to 3 replicas
5. Wait for the new pod to become ready.
6. Verify the new volume properties.
"""
statefulset_name = 'statefulset-scaling-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
volumes = client.list_volume()
assert len(volumes) == statefulset['spec']['replicas']
for v in volumes:
found = False
for pod in pod_info:
if v.name == pod['pv_name']:
found = True
break
assert found
pod_info.remove(pod)
assert v.size == str(DEFAULT_VOLUME_SIZE * Gi)
assert v.numberOfReplicas == \
int(storage_class['parameters']['numberOfReplicas'])
assert v.state == 'attached'
assert len(pod_info) == 0
statefulset['spec']['replicas'] = replicas = 3
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={'spec': {
'replicas': replicas
}})
for i in range(DEFAULT_POD_TIMEOUT):
s_set = apps_api.read_namespaced_stateful_set(name=statefulset_name,
namespace='default')
if s_set.status.ready_replicas == replicas:
break
time.sleep(DEFAULT_POD_INTERVAL)
assert s_set.status.ready_replicas == replicas
pod_info = get_statefulset_pod_info(core_api, statefulset)
volumes = client.list_volume()
assert len(volumes) == replicas
for v in volumes:
found = False
for pod in pod_info:
if v.name == pod['pv_name']:
found = True
break
assert found
pod_info.remove(pod)
assert v.size == str(DEFAULT_VOLUME_SIZE * Gi)
assert v.numberOfReplicas == \
int(storage_class['parameters']['numberOfReplicas'])
assert v.state == 'attached'
assert len(pod_info) == 0
def test_offline_node_with_attached_volume_and_pod(
client, core_api, volume_name, make_deployment_with_pvc,
reset_cluster_ready_status): # NOQA
"""
Test offline node with attached volume and pod
1. Create PV/PVC/Deployment manifest.
2. Update deployment's tolerations to 20 seconds to speed up test
3. Update deployment's node affinity rule to avoid the current node
4. Create volume, PV/PVC and deployment.
5. Find the pod in the deployment and write `test_data` into it
6. Shutdown the node pod is running on
7. Wait for deployment to delete the pod
1. Deployment cannot delete the pod here because kubelet doesn't
response
8. Force delete the terminating pod
9. Wait for the new pod to be created and the volume attached
10. Check `test_data` in the new pod
"""
toleration_seconds = 20
apps_api = get_apps_api_client()
cloudprovider = detect_cloudprovider()
volume_name = generate_volume_name()
pv_name = volume_name + "-pv"
pvc_name = volume_name + "-pvc"
deployment_name = volume_name + "-dep"
longhorn_test_node_name = get_self_host_id()
deployment_manifest = make_deployment_with_pvc(deployment_name, pvc_name)
unreachable_toleration = {
"key": "node.kubernetes.io/unreachable",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": toleration_seconds
}
not_ready_toleration = {
"key": "node.kubernetes.io/not-ready",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": toleration_seconds
}
deployment_manifest["spec"]["template"]["spec"]["tolerations"] =\
[unreachable_toleration, not_ready_toleration]
node_affinity_roles = {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kubernetes.io/hostname",
"operator": "NotIn",
"values": [longhorn_test_node_name]
}]
}]
}
}
}
deployment_manifest["spec"]["template"]["spec"]["affinity"] =\
node_affinity_roles
longhorn_volume = create_and_check_volume(client, volume_name, size=SIZE)
wait_for_volume_detached(client, volume_name)
create_pv_for_volume(client, core_api, longhorn_volume, pv_name)
create_pvc_for_volume(client, core_api, longhorn_volume, pvc_name)
create_and_wait_deployment(apps_api, deployment_manifest)
deployment_label_selector =\
"name=" + deployment_manifest["metadata"]["labels"]["name"]
deployment_pod_list =\
core_api.list_namespaced_pod(namespace="default",
label_selector=deployment_label_selector)
assert deployment_pod_list.items.__len__() == 1
pod_name = deployment_pod_list.items[0].metadata.name
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
node_name = deployment_pod_list.items[0].spec.node_name
node = cloudprovider.node_id(node_name)
cloudprovider.node_shutdown(node)
k8s_node_down = wait_for_node_down_k8s(node_name, core_api)
assert k8s_node_down
client = get_longhorn_api_client()
longhorn_node_down = wait_for_node_down_longhorn(node_name, client)
assert longhorn_node_down
time.sleep(toleration_seconds + 5)
for i in range(TERMINATING_POD_RETRYS):
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
terminating_pod_name = None
for pod in deployment_pod_list.items:
if pod.metadata.__getattribute__("deletion_timestamp") is not None:
terminating_pod_name = pod.metadata.name
break
if terminating_pod_name is not None:
break
else:
time.sleep(TERMINATING_POD_INTERVAL)
assert terminating_pod_name is not None
core_api.delete_namespaced_pod(namespace="default",
name=terminating_pod_name,
grace_period_seconds=0)
delete_and_wait_pod(core_api, terminating_pod_name)
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
assert deployment_pod_list.items.__len__() == 1
wait_for_volume_detached(client, volume_name)
wait_for_volume_healthy(client, volume_name)
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
assert deployment_pod_list.items.__len__() == 1
new_pod_name = deployment_pod_list.items[0].metadata.name
wait_pod(new_pod_name)
resp_data = read_volume_data(core_api, new_pod_name)
assert test_data == resp_data
def test_restore_rwo_volume_to_rwx(set_random_backupstore, client, core_api,
volume_name, pvc, csi_pv, pod_make,
make_deployment_with_pvc): # NOQA
"""
Test restoring a rwo to a rwx volume.
1. Create a volume with 'accessMode' rwo.
2. Create a PV and a PVC with access mode 'readwriteonce' and attach to the
volume.
3. Create a pod and attach to the PVC.
4. Write some data into the pod and compute md5sum.
5. Take a backup of the volume.
6. Restore the backup with 'accessMode' rwx.
7. Create PV and PVC and attach to 2 pods.
8. Verify the data.
"""
data_path = "/data/test"
pod_name, pv_name, pvc_name, md5sum = \
prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc,
pod_make,
volume_name,
data_size_in_mb=DATA_SIZE_IN_MB_1,
data_path=data_path)
snap = create_snapshot(client, volume_name)
volume = client.by_id_volume(volume_name)
volume.snapshotBackup(name=snap.name)
wait_for_backup_completion(client, volume_name, snap.name)
bv, b1 = find_backup(client, volume_name, snap.name)
restore_volume_name = 'restored-rwx-volume'
restore_pv_name = restore_volume_name + "-pv"
restore_pvc_name = restore_volume_name + "-pvc"
client.create_volume(name=restore_volume_name,
size=str(1 * Gi),
numberOfReplicas=3,
fromBackup=b1.url,
accessMode='rwx')
wait_for_volume_creation(client, restore_volume_name)
restore_volume = wait_for_volume_detached(client, restore_volume_name)
create_pv_for_volume(client, core_api, restore_volume, restore_pv_name)
create_pvc_for_volume(client, core_api, restore_volume, restore_pvc_name)
deployment = make_deployment_with_pvc('deployment-multi-pods-test',
restore_pvc_name,
replicas=2)
apps_api = get_apps_api_client()
create_and_wait_deployment(apps_api, deployment)
deployment_label_selector = \
"name=" + deployment["metadata"]["labels"]["name"]
deployment_pod_list = \
core_api.list_namespaced_pod(namespace="default",
label_selector=deployment_label_selector)
pod_name_1 = deployment_pod_list.items[0].metadata.name
pod_name_2 = deployment_pod_list.items[1].metadata.name
md5sum_pod1 = get_pod_data_md5sum(core_api, pod_name_1, data_path)
md5sum_pod2 = get_pod_data_md5sum(core_api, pod_name_2, data_path)
assert md5sum == md5sum_pod1 == md5sum_pod2
def test_rwx_deployment_with_multi_pods(core_api, pvc,
make_deployment_with_pvc): # NOQA
"""
Test deployment of 2 pods with same PVC.
1. Create a volume with 'accessMode' rwx.
2. Create a PV and a PVC with access mode 'readwritemany' and attach to the
volume.
3. Create a deployment of 2 pods with PVC created
4. Wait for 2 pods to come up healthy.
5. Write data in both pods and compute md5sum.
6. Check the data md5sum in the share manager pod.
"""
pvc_name = 'pvc-deployment-multi-pods-test'
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = 'longhorn'
pvc['spec']['accessModes'] = ['ReadWriteMany']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
deployment = make_deployment_with_pvc('deployment-multi-pods-test',
pvc_name,
replicas=2)
apps_api = get_apps_api_client()
create_and_wait_deployment(apps_api, deployment)
pv_name = get_volume_name(core_api, pvc_name)
share_manager_name = 'share-manager-' + pv_name
deployment_label_selector = "name=" + \
deployment["metadata"]["labels"]["name"]
deployment_pod_list = \
core_api.list_namespaced_pod(namespace="default",
label_selector=deployment_label_selector)
assert deployment_pod_list.items.__len__() == 2
pod_name_1 = deployment_pod_list.items[0].metadata.name
test_data_1 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name_1, test_data_1, filename='test1')
pod_name_2 = deployment_pod_list.items[1].metadata.name
command = 'cat /data/test1'
pod_data_2 = exec_command_in_pod(core_api, command, pod_name_2, 'default')
assert test_data_1 == pod_data_2
test_data_2 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name_2, test_data_2, filename='test2')
command = 'cat /export' + '/' + pv_name + '/' + 'test1'
share_manager_data_1 = exec_command_in_pod(core_api, command,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_1 == share_manager_data_1
command = 'cat /export' + '/' + pv_name + '/' + 'test2'
share_manager_data_2 = exec_command_in_pod(core_api, command,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_2 == share_manager_data_2
def test_rwx_statefulset_scale_down_up(core_api, statefulset): # NOQA
"""
Test Scaling up and down of pods attached to rwx volume.
1. Create a StatefulSet of 2 pods with VolumeClaimTemplate where accessMode
is 'RWX'.
2. Wait for StatefulSet pods to come up healthy.
3. Write data and compute md5sum in the both pods.
4. Delete the pods.
5. Wait for the pods to be terminated.
6. Verify the share manager pods are no longer available and the volume is
detached.
6. Recreate the pods
7. Wait for new pods to come up.
8. Check the data md5sum in new pods.
"""
statefulset_name = 'statefulset-rwx-scale-down-up-test'
share_manager_name = []
statefulset['metadata']['name'] = \
statefulset['spec']['selector']['matchLabels']['app'] = \
statefulset['spec']['serviceName'] = \
statefulset['spec']['template']['metadata']['labels']['app'] = \
statefulset_name
statefulset['spec']['volumeClaimTemplates'][0]['spec']['storageClassName']\
= 'longhorn'
statefulset['spec']['volumeClaimTemplates'][0]['spec']['accessModes'] \
= ['ReadWriteMany']
create_and_wait_statefulset(statefulset)
for i in range(2):
pvc_name = \
statefulset['spec']['volumeClaimTemplates'][0]['metadata']['name']\
+ '-' + statefulset_name + '-' + str(i)
pv_name = get_volume_name(core_api, pvc_name)
assert pv_name is not None
share_manager_name.append('share-manager-' + pv_name)
check_pod_existence(core_api,
share_manager_name[i],
namespace=LONGHORN_NAMESPACE)
md5sum_pod = []
for i in range(2):
test_pod_name = statefulset_name + '-' + str(i)
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, test_pod_name, test_data)
md5sum_pod.append(test_data)
statefulset['spec']['replicas'] = replicas = 0
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={'spec': {
'replicas': replicas
}})
for i in range(DEFAULT_STATEFULSET_TIMEOUT):
s_set = apps_api.read_namespaced_stateful_set(
name=statefulset['metadata']['name'], namespace='default')
if s_set.status.ready_replicas == replicas or \
(replicas == 0 and not s_set.status.ready_replicas):
break
time.sleep(DEFAULT_STATEFULSET_INTERVAL)
pods = core_api.list_namespaced_pod(namespace=LONGHORN_NAMESPACE)
found = False
for item in pods.items:
if item.metadata.name == share_manager_name[0] or \
item.metadata.name == share_manager_name[1]:
found = True
break
assert not found
statefulset['spec']['replicas'] = replicas = 2
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={'spec': {
'replicas': replicas
}})
wait_statefulset(statefulset)
for i in range(2):
test_pod_name = statefulset_name + '-' + str(i)
command = 'cat /data/test'
pod_data = exec_command_in_pod(core_api, command, test_pod_name,
'default')
assert pod_data == md5sum_pod[i]
def test_upgrade(upgrade_image_tag, settings_reset, volume_name, pod_make, statefulset, storage_class): # NOQA
"""
Test Longhorn upgrade
Prerequisite:
- Disable Auto Salvage Setting
1. Find the upgrade image tag
2. Create a volume, generate and write data into the volume.
3. Create a Pod using a volume, generate and write data
4. Create a StatefulSet with 2 replicas,
generate and write data to their volumes
5. Keep all volumes attached
6. Upgrade Longhorn system.
7. Check Pod and StatefulSet didn't restart after upgrade
8. Check All volumes data
9. Write data to StatefulSet pods, and Attached volume
10. Check data written to StatefulSet pods, and attached volume.
11. Detach the volume, and Delete Pod, and
StatefulSet to detach theirvolumes
12. Upgrade all volumes engine images.
13. Attach the volume, and recreate Pod, and StatefulSet
14. Check All volumes data
"""
new_ei_name = "longhornio/longhorn-engine:" + upgrade_image_tag
client = get_longhorn_api_client()
core_api = get_core_api_client()
host_id = get_self_host_id()
pod_data_path = "/data/test"
pod_volume_name = generate_volume_name()
auto_salvage_setting = client.by_id_setting(SETTING_AUTO_SALVAGE)
setting = client.update(auto_salvage_setting, value="false")
assert setting.name == SETTING_AUTO_SALVAGE
assert setting.value == "false"
# Create Volume attached to a node.
volume1 = create_and_check_volume(client,
volume_name,
size=SIZE)
volume1.attach(hostId=host_id)
volume1 = wait_for_volume_healthy(client, volume_name)
volume1_data = write_volume_random_data(volume1)
# Create Volume used by Pod
pod_name, pv_name, pvc_name, pod_md5sum = \
prepare_pod_with_data_in_mb(client, core_api,
pod_make, pod_volume_name,
data_path=pod_data_path,
add_liveness_prope=False)
# Create multiple volumes used by StatefulSet
statefulset_name = 'statefulset-upgrade-test'
update_statefulset_manifests(statefulset,
storage_class,
statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
statefulset_pod_info = get_statefulset_pod_info(core_api, statefulset)
for sspod_info in statefulset_pod_info:
sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api,
sspod_info['pod_name'],
sspod_info['data'])
# upgrade Longhorn
assert longhorn_upgrade(upgrade_image_tag)
client = get_longhorn_api_client()
# wait for 1 minute before checking pod restarts
time.sleep(60)
pod = core_api.read_namespaced_pod(name=pod_name,
namespace='default')
assert pod.status.container_statuses[0].restart_count == 0
for sspod_info in statefulset_pod_info:
sspod = core_api.read_namespaced_pod(name=sspod_info['pod_name'],
namespace='default')
assert \
sspod.status.container_statuses[0].restart_count == 0
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
assert res_pod_md5sum == pod_md5sum
check_volume_data(volume1, volume1_data)
for sspod_info in statefulset_pod_info:
sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api,
sspod_info['pod_name'],
sspod_info['data'])
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
volume1 = client.by_id_volume(volume_name)
volume1_data = write_volume_random_data(volume1)
check_volume_data(volume1, volume1_data)
statefulset['spec']['replicas'] = replicas = 0
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={
'spec': {
'replicas': replicas
}
})
delete_and_wait_pod(core_api, pod_name)
volume = client.by_id_volume(volume_name)
volume.detach()
volumes = client.list_volume()
for v in volumes:
wait_for_volume_detached(client, v.name)
engineimages = client.list_engine_image()
for ei in engineimages:
if ei.image == new_ei_name:
new_ei = ei
volumes = client.list_volume()
for v in volumes:
volume = client.by_id_volume(v.name)
volume.engineUpgrade(image=new_ei.image)
statefulset['spec']['replicas'] = replicas = 2
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={
'spec': {
'replicas': replicas
}
})
wait_statefulset(statefulset)
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
create_and_wait_pod(core_api, pod)
volume1 = client.by_id_volume(volume_name)
volume1.attach(hostId=host_id)
volume1 = wait_for_volume_healthy(client, volume_name)
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
assert res_pod_md5sum == pod_md5sum
check_volume_data(volume1, volume1_data)
def test_setting_toleration():
"""
Test toleration setting
1. Verify that cannot use Kubernetes tolerations for Longhorn setting
2. Use "key1=value1:NoSchedule; key2:NoExecute" as toleration.
3. Create a volume and attach it.
4. Verify that cannot update toleration setting when any volume is attached
5. Generate and write `data1` into the volume
6. Detach the volume.
7. Update setting `toleration` to toleration.
8. Wait for all the Longhorn components to restart with new toleration
9. Attach the volume again and verify the volume `data1`.
10. Generate and write `data2` to the volume.
11. Detach the volume.
12. Clean the `toleration` setting.
13. Wait for all the Longhorn components to restart with no toleration
14. Attach the volume and validate `data2`.
15. Generate and write `data3` to the volume.
"""
client = get_longhorn_api_client() # NOQA
apps_api = get_apps_api_client() # NOQA
core_api = get_core_api_client() # NOQA
count = len(client.list_node())
setting = client.by_id_setting(SETTING_TAINT_TOLERATION)
with pytest.raises(Exception) as e:
client.update(setting,
value=KUBERNETES_DEFAULT_TOLERATION + ":NoSchedule")
assert "is considered as the key of Kubernetes default tolerations" \
in str(e.value)
with pytest.raises(Exception) as e:
client.update(setting,
value="key1=value1:NoSchedule; key2:InvalidEffect")
assert 'invalid effect' in str(e.value)
setting_value_str = "key1=value1:NoSchedule; key2:NoExecute"
setting_value_dict = \
{"key1": {"key": "key1", "value": "value1",
"operator": "Equal", "effect": "NoSchedule"},
"key2": {"key": "key2", "value": None,
"operator": "Exists", "effect": "NoExecute"}, }
volume_name = "test-toleration-vol" # NOQA
volume = create_and_check_volume(client, volume_name)
volume.attach(hostId=get_self_host_id())
volume = wait_for_volume_healthy(client, volume_name)
with pytest.raises(Exception) as e:
client.update(setting, value=setting_value_str)
assert 'cannot modify toleration setting before all volumes are detached' \
in str(e.value)
data1 = write_volume_random_data(volume)
check_volume_data(volume, data1)
volume.detach()
wait_for_volume_detached(client, volume_name)
setting = client.update(setting, value=setting_value_str)
assert setting.value == setting_value_str
wait_for_toleration_update(core_api, apps_api, count, setting_value_dict)
client, node = wait_for_longhorn_node_ready()
volume = client.by_id_volume(volume_name)
volume.attach(hostId=node)
volume = wait_for_volume_healthy(client, volume_name)
check_volume_data(volume, data1)
data2 = write_volume_random_data(volume)
check_volume_data(volume, data2)
volume.detach()
wait_for_volume_detached(client, volume_name)
# cleanup
setting_value_str = ""
setting_value_dict = {}
setting = client.by_id_setting(SETTING_TAINT_TOLERATION)
setting = client.update(setting, value=setting_value_str)
assert setting.value == setting_value_str
wait_for_toleration_update(core_api, apps_api, count, setting_value_dict)
client, node = wait_for_longhorn_node_ready()
volume = client.by_id_volume(volume_name)
volume.attach(hostId=node)
volume = wait_for_volume_healthy(client, volume_name)
check_volume_data(volume, data2)
data3 = write_volume_random_data(volume)
check_volume_data(volume, data3)
cleanup_volume(client, volume)
def test_statefulset_scaling(client, core_api, storage_class, statefulset): # NOQA
"""
Test that scaling up a StatefulSet successfully provisions new volumes.
"""
statefulset_name = 'statefulset-scaling-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
volumes = client.list_volume()
assert len(volumes) == statefulset['spec']['replicas']
for v in volumes:
found = False
for pod in pod_info:
if v['name'] == pod['pv_name']:
found = True
break
assert found
pod_info.remove(pod)
assert v['size'] == str(DEFAULT_VOLUME_SIZE * Gi)
assert v['numberOfReplicas'] == \
int(storage_class['parameters']['numberOfReplicas'])
assert v['state'] == 'attached'
assert len(pod_info) == 0
statefulset['spec']['replicas'] = replicas = 3
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={
'spec': {
'replicas': replicas
}
})
for i in range(DEFAULT_POD_TIMEOUT):
s_set = apps_api.read_namespaced_stateful_set(
name=statefulset_name,
namespace='default')
if s_set.status.ready_replicas == replicas:
break
time.sleep(DEFAULT_POD_INTERVAL)
assert s_set.status.ready_replicas == replicas
pod_info = get_statefulset_pod_info(core_api, statefulset)
volumes = client.list_volume()
assert len(volumes) == replicas
for v in volumes:
found = False
for pod in pod_info:
if v['name'] == pod['pv_name']:
found = True
break
assert found
pod_info.remove(pod)
assert v['size'] == str(DEFAULT_VOLUME_SIZE * Gi)
assert v['numberOfReplicas'] == \
int(storage_class['parameters']['numberOfReplicas'])
assert v['state'] == 'attached'
assert len(pod_info) == 0
