def guaranteed_engine_cpu_setting_check( # NOQA
client, core_api, instance_managers, state, desire, cpu_val): # NOQA
"""
We check if instance managers are in the desired state with
correct setting
desire is for reflect the state we are looking for.
If desire is True, meanning we need the state to be the same.
Otherwise, we are looking for the state to be different.
e.g. 'Pending', 'OutofCPU', 'Terminating' they are all 'Not Running'.
"""
# Give sometime to k8s to update the instance manager status
for im in instance_managers:
wait_for_instance_manager_desire_state(client, core_api, im.name,
state, desire)
if desire:
# Verify guaranteed CPU set correctly
for im in instance_managers:
pod = core_api.read_namespaced_pod(name=im.name,
namespace=LONGHORN_NAMESPACE)
if cpu_val:
assert (pod.spec.containers[0].resources.requests['cpu'] ==
cpu_val)
else:
assert (not pod.spec.containers[0].resources.requests)
def test_kubernetes_status(
client,
core_api,
storage_class, # NOQA
statefulset,
csi_pv,
pvc,
pod): # NOQA
"""
Test Volume feature: Kubernetes Status
1. Create StorageClass with `reclaimPolicy = Retain`
2. Create a statefulset `kubernetes-status-test` with the StorageClass
1. The statefulset has scale of 2.
3. Get the volume name from the SECOND pod of the StateufulSet pod and
create an `extra_pod` with the same volume on the same node
4. Check the volumes that used by the StatefulSet
1. The volume used by the FIRST StatefulSet pod will have one workload
2. The volume used by the SECOND StatefulSet pod will have two
workloads
3. Validate related status, e.g. pv/pod name/state, workload
name/type
5. Check the volumes again
1. PV/PVC should still be bound
2. The volume used by the FIRST pod should have history data
3. The volume used by the SECOND and extra pod should have current data
point to the extra pod
6. Delete the extra pod
1. Now all the volume's should only have history data(`lastPodRefAt`
set)
7. Delete the PVC
1. PVC should be updated with status `Released` and become history data
8. Delete PV
1. All the Kubernetes status information should be cleaned up.
9. Reuse the two Longhorn volumes to create new pods
1. Since the `reclaimPolicy == Retain`, volume won't be deleted by
Longhorn
2. Check the Kubernetes status now updated, with pod info but empty
workload
3. Default Longhorn Static StorageClass will remove the PV with PVC,
but leave Longhorn volume
"""
statefulset_name = 'kubernetes-status-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
storage_class['reclaimPolicy'] = 'Retain'
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
volume_info = [p['pv_name'] for p in pod_info]
extra_pod_name = 'extra-pod-using-' + volume_info[1]
pod['metadata']['name'] = extra_pod_name
p2 = core_api.read_namespaced_pod(name=pod_info[1]['pod_name'],
namespace='default')
pod['spec']['nodeName'] = p2.spec.node_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pod_info[1]['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i] # NOQA
volume = client.by_id_volume(volume_name)
k_status = volume.kubernetesStatus
workloads = k_status.workloadsStatus
assert k_status.pvName == p['pv_name']
assert k_status.pvStatus == 'Bound'
assert k_status.namespace == 'default'
assert k_status.pvcName == p['pvc_name']
assert not k_status.lastPVCRefAt
assert not k_status.lastPodRefAt
if i == 0:
assert len(workloads) == 1
assert workloads[0].podName == p['pod_name']
assert workloads[0].workloadName == statefulset_name
assert workloads[0].workloadType == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0].podStatus == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume.kubernetesStatus
workloads = k_status.workloadsStatus
assert workloads[0].podStatus == 'Running'
if i == 1:
assert len(k_status.workloadsStatus) == 2
if workloads[0].podName == pod_info[i]['pod_name']:
assert workloads[1].podName == extra_pod_name
assert workloads[0].workloadName == statefulset_name
assert workloads[0].workloadType == 'StatefulSet'
assert not workloads[1].workloadName
assert not workloads[1].workloadType
else:
assert workloads[1].podName == pod_info[i]['pod_name']
assert workloads[0].podName == extra_pod_name
assert not workloads[0].workloadName
assert not workloads[0].workloadType
assert workloads[1].workloadName == statefulset_name
assert workloads[1].workloadType == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0].podStatus == 'Running' and \
workloads[1].podStatus == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume.kubernetesStatus
workloads = k_status.workloadsStatus
assert len(workloads) == 2
assert workloads[0].podStatus == 'Running'
assert workloads[1].podStatus == 'Running'
ks_list = [{}, {}]
delete_and_wait_statefulset_only(core_api, statefulset)
# the extra pod is still using the 2nd volume
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
ks_list[i]['pvName'] = p['pv_name']
ks_list[i]['pvStatus'] = 'Bound'
ks_list[i]['namespace'] = 'default'
ks_list[i]['pvcName'] = p['pvc_name']
ks_list[i]['lastPVCRefAt'] = ''
if i == 0:
ks_list[i]['lastPodRefAt'] = 'not empty'
ks_list[i]['workloadsStatus'] = [
{
'podName': p['pod_name'],
'podStatus': 'Running',
'workloadName': statefulset_name,
'workloadType': 'StatefulSet',
},
]
if i == 1:
ks_list[i]['lastPodRefAt'] = ''
ks_list[i]['workloadsStatus'] = [{
'podName': extra_pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
}]
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted extra_pod, all volumes have no workload
delete_and_wait_pod(core_api, pod['metadata']['name'])
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
ks_list[i]['lastPodRefAt'] = 'not empty'
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted pvc only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pvc(core_api, p['pvc_name'])
ks_list[i]['pvStatus'] = 'Released'
ks_list[i]['lastPVCRefAt'] = 'not empty'
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted pv only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pv(core_api, p['pv_name'])
ks_list[i]['pvName'] = ''
ks_list[i]['pvStatus'] = ''
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# reuse that volume
for p, volume_name in zip(pod_info, volume_info):
p['pod_name'] = p['pod_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pvc_name'] = p['pvc_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pv_name'] = p['pvc_name']
csi_pv['metadata']['name'] = p['pv_name']
csi_pv['spec']['csi']['volumeHandle'] = volume_name
csi_pv['spec']['storageClassName'] = \
DEFAULT_LONGHORN_STATIC_STORAGECLASS_NAME
core_api.create_persistent_volume(csi_pv)
pvc['metadata']['name'] = p['pvc_name']
pvc['spec']['volumeName'] = p['pv_name']
pvc['spec']['storageClassName'] = \
DEFAULT_LONGHORN_STATIC_STORAGECLASS_NAME
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
pod['metadata']['name'] = p['pod_name']
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': p['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'pvName':
p['pv_name'],
'pvStatus':
'Bound',
'namespace':
'default',
'pvcName':
p['pvc_name'],
'lastPVCRefAt':
'',
'lastPodRefAt':
'',
'workloadsStatus': [
{
'podName': p['pod_name'],
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
},
],
}
wait_volume_kubernetes_status(client, volume_name, ks)
delete_and_wait_pod(core_api, p['pod_name'])
# Since persistentVolumeReclaimPolicy of csi_pv is `Delete`,
# we don't need to delete bounded pv manually
delete_and_wait_pvc(core_api, p['pvc_name'])
wait_delete_pv(core_api, p['pv_name'])
def test_rwx_delete_share_manager_pod(core_api, statefulset): # NOQA
"""
Test moving of Share manager pod from one node to another.
1. Create a StatefulSet of 1 pod with VolumeClaimTemplate where accessMode
is 'RWX'.
2. Wait for StatefulSet to come up healthy.
3. Write data and compute md5sum.
4. Delete the share manager pod.
5. Wait for a new pod to be created and volume getting attached.
6. Check the data md5sum in statefulSet.
7. Write more data to it and compute md5sum.
8. Check the data md5sum in share manager volume.
"""
statefulset_name = 'statefulset-delete-share-manager-pods-test'
statefulset['metadata']['name'] = \
statefulset['spec']['selector']['matchLabels']['app'] = \
statefulset['spec']['serviceName'] = \
statefulset['spec']['template']['metadata']['labels']['app'] = \
statefulset_name
statefulset['spec']['replicas'] = 1
statefulset['spec']['volumeClaimTemplates'][0]['spec']['storageClassName']\
= 'longhorn'
statefulset['spec']['volumeClaimTemplates'][0]['spec']['accessModes'] \
= ['ReadWriteMany']
create_and_wait_statefulset(statefulset)
pod_name = statefulset_name + '-' + '0'
pvc_name = \
statefulset['spec']['volumeClaimTemplates'][0]['metadata']['name'] \
+ '-' + statefulset_name + '-0'
pv_name = get_volume_name(core_api, pvc_name)
share_manager_name = 'share-manager-' + pv_name
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data, filename='test1')
delete_and_wait_pod(core_api,
share_manager_name,
namespace=LONGHORN_NAMESPACE)
target_pod = core_api.read_namespaced_pod(name=pod_name,
namespace='default')
wait_delete_pod(core_api, target_pod.metadata.uid)
wait_for_pod_remount(core_api, pod_name)
test_data_2 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data_2, filename='test2')
command1 = 'cat /export/' + pv_name + '/test1'
share_manager_data_1 = exec_command_in_pod(core_api, command1,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data == share_manager_data_1
command2 = 'cat /export/' + pv_name + '/test2'
share_manager_data_2 = exec_command_in_pod(core_api, command2,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_2 == share_manager_data_2
def test_kubernetes_status(client, core_api, storage_class, # NOQA
statefulset, csi_pv, pvc, pod): # NOQA
statefulset_name = 'kubernetes-status-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
storage_class['reclaimPolicy'] = 'Retain'
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
volume_info = [p['pv_name'] for p in pod_info]
extra_pod_name = 'extra-pod-using-' + volume_info[1]
pod['metadata']['name'] = extra_pod_name
p2 = core_api.read_namespaced_pod(name=pod_info[1]['pod_name'],
namespace='default')
pod['spec']['nodeName'] = p2.spec.node_name
pod['spec']['volumes'] = [{
'name': pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pod_info[1]['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == p['pv_name']
assert k_status['pvStatus'] == 'Bound'
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert not k_status['lastPVCRefAt']
assert not k_status['lastPodRefAt']
if i == 0:
assert len(workloads) == 1
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert workloads[0]['podStatus'] == 'Running'
if i == 1:
assert len(k_status['workloadsStatus']) == 2
if workloads[0]['podName'] == pod_info[i]['pod_name']:
assert workloads[1]['podName'] == extra_pod_name
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
assert not workloads[1]['workloadName']
assert not workloads[1]['workloadType']
else:
assert workloads[1]['podName'] == pod_info[i]['pod_name']
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
assert workloads[1]['workloadName'] == statefulset_name
assert workloads[1]['workloadType'] == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running' and \
workloads[1]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 2
assert workloads[0]['podStatus'] == 'Running'
assert workloads[1]['podStatus'] == 'Running'
# the extra pod is still using the 2nd volume
delete_and_wait_statefulset_only(core_api, statefulset)
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == p['pv_name']
assert k_status['pvStatus'] == 'Bound'
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert not k_status['lastPVCRefAt']
assert len(workloads) == 1
if i == 0:
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
assert k_status['lastPodRefAt']
if i == 1:
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
assert not k_status['lastPodRefAt']
# deleted extra_pod, all volumes have no workload
delete_and_wait_pod(core_api, pod['metadata']['name'])
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == p['pv_name']
assert k_status['pvStatus'] == 'Bound'
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert not k_status['lastPVCRefAt']
assert k_status['lastPodRefAt']
assert len(workloads) == 1
if i == 0:
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
if i == 1:
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
# deleted pvc only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pvc(core_api, p['pvc_name'])
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
for _ in range(RETRY_COUNTS):
if k_status['pvStatus'] == 'Released':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == p['pv_name']
assert k_status['pvStatus'] == 'Released'
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert k_status['lastPVCRefAt']
assert k_status['lastPodRefAt']
assert len(workloads) == 1
if i == 0:
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
if i == 1:
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
# deleted pv only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pv(core_api, p['pv_name'])
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == ''
assert k_status['pvStatus'] == ''
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert k_status['lastPVCRefAt']
assert k_status['lastPodRefAt']
assert len(workloads) == 1
if i == 0:
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
if i == 1:
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
# reuse that volume
for p, volume_name in zip(pod_info, volume_info):
p['pod_name'] = p['pod_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pvc_name'] = p['pvc_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pv_name'] = p['pvc_name']
csi_pv['metadata']['name'] = p['pv_name']
csi_pv['spec']['csi']['volumeHandle'] = volume_name
core_api.create_persistent_volume(csi_pv)
pvc['metadata']['name'] = p['pvc_name']
pvc['spec']['volumeName'] = p['pv_name']
core_api.create_namespaced_persistent_volume_claim(
body=pvc, namespace='default')
pod['metadata']['name'] = p['pod_name']
pod['spec']['volumes'] = [{
'name': pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': p['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 1
assert k_status['pvName'] == p['pv_name']
for _ in range(RETRY_COUNTS):
if k_status['pvStatus'] == 'Bound':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 1
assert k_status['pvStatus'] == 'Bound'
for _ in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 1
assert workloads[0]['podStatus'] == 'Running'
assert workloads[0]['podName'] == p['pod_name']
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert not k_status['lastPVCRefAt']
assert not k_status['lastPodRefAt']
delete_and_wait_pod(core_api, p['pod_name'])
# Since persistentVolumeReclaimPolicy of csi_pv is `Delete`,
# we don't need to delete bounded pv manually
delete_and_wait_pvc(core_api, p['pvc_name'])
wait_delete_pv(core_api, p['pv_name'])
def test_upgrade(
upgrade_longhorn_manager_repo_url,
upgrade_longhorn_manager_repo_branch,
upgrade_longhorn_manager_image,
upgrade_longhorn_engine_image,
upgrade_longhorn_instance_manager_image,
upgrade_longhorn_share_manager_image,
upgrade_longhorn_backing_image_manager_image,
client,
core_api,
volume_name,
csi_pv, # NOQA
pvc,
pod_make,
statefulset,
storage_class): # NOQA
"""
Test Longhorn upgrade
TODO
The test will cover both volume has revision counter enabled and
disabled cases.
Prerequisite:
- Disable Auto Salvage Setting
1. Find the upgrade image tag
2. Create a volume, generate and write data into the volume.
1. Create a volume with revision counter enabled case.
2. Create a volume with revision counter disabled case.
3. Create a Pod using a volume, generate and write data
4. Create a StatefulSet with 2 replicas,
generate and write data to their volumes
5. Keep all volumes attached
6. Upgrade Longhorn system.
7. Check Pod and StatefulSet didn't restart after upgrade
8. Check All volumes data
9. Write data to StatefulSet pods, and Attached volume
10. Check data written to StatefulSet pods, and attached volume.
11. Detach the volume, and Delete Pod, and
StatefulSet to detach theirvolumes
12. Upgrade all volumes engine images.
13. Attach the volume, and recreate Pod, and StatefulSet
14. Check All volumes data
"""
longhorn_manager_repo = upgrade_longhorn_manager_repo_url
longhorn_manager_branch = upgrade_longhorn_manager_repo_branch
longhorn_manager_image = upgrade_longhorn_manager_image
longhorn_engine_image = upgrade_longhorn_engine_image
longhorn_instance_manager_image = upgrade_longhorn_instance_manager_image
longhorn_share_manager_image = upgrade_longhorn_share_manager_image
longhorn_backing_image_manager_image = \
upgrade_longhorn_backing_image_manager_image
host_id = get_self_host_id()
pod_data_path = "/data/test"
pod_volume_name = generate_volume_name()
auto_salvage_setting = client.by_id_setting(SETTING_AUTO_SALVAGE)
setting = client.update(auto_salvage_setting, value="false")
assert setting.name == SETTING_AUTO_SALVAGE
assert setting.value == "false"
# Create Volume attached to a node.
volume1 = create_and_check_volume(client, volume_name, size=SIZE)
volume1.attach(hostId=host_id)
volume1 = wait_for_volume_healthy(client, volume_name)
volume1_data = write_volume_random_data(volume1)
# Create Volume used by Pod
pod_name, pv_name, pvc_name, pod_md5sum = \
prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc,
pod_make, pod_volume_name,
data_path=pod_data_path,
add_liveness_probe=False)
# Create multiple volumes used by StatefulSet
statefulset_name = 'statefulset-upgrade-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
statefulset_pod_info = get_statefulset_pod_info(core_api, statefulset)
for sspod_info in statefulset_pod_info:
sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, sspod_info['pod_name'],
sspod_info['data'])
# upgrade Longhorn
assert longhorn_upgrade(longhorn_manager_repo, longhorn_manager_branch,
longhorn_manager_image, longhorn_engine_image,
longhorn_instance_manager_image,
longhorn_share_manager_image,
longhorn_backing_image_manager_image)
client = get_longhorn_api_client()
# wait for 1 minute before checking pod restarts
time.sleep(60)
pod = core_api.read_namespaced_pod(name=pod_name, namespace='default')
assert pod.status.container_statuses[0].restart_count == 0
for sspod_info in statefulset_pod_info:
sspod = core_api.read_namespaced_pod(name=sspod_info['pod_name'],
namespace='default')
assert \
sspod.status.container_statuses[0].restart_count == 0
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
assert res_pod_md5sum == pod_md5sum
check_volume_data(volume1, volume1_data)
for sspod_info in statefulset_pod_info:
sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, sspod_info['pod_name'],
sspod_info['data'])
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
volume1 = client.by_id_volume(volume_name)
volume1_data = write_volume_random_data(volume1)
check_volume_data(volume1, volume1_data)
statefulset['spec']['replicas'] = replicas = 0
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={'spec': {
'replicas': replicas
}})
delete_and_wait_pod(core_api, pod_name)
volume = client.by_id_volume(volume_name)
volume.detach(hostId="")
volumes = client.list_volume()
for v in volumes:
wait_for_volume_detached(client, v.name)
engineimages = client.list_engine_image()
for ei in engineimages:
if ei.image == longhorn_engine_image:
new_ei = ei
volumes = client.list_volume()
for v in volumes:
volume = client.by_id_volume(v.name)
volume.engineUpgrade(image=new_ei.image)
statefulset['spec']['replicas'] = replicas = 2
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={'spec': {
'replicas': replicas
}})
wait_statefulset(statefulset)
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
create_and_wait_pod(core_api, pod)
volume1 = client.by_id_volume(volume_name)
volume1.attach(hostId=host_id)
volume1 = wait_for_volume_healthy(client, volume_name)
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
assert res_pod_md5sum == pod_md5sum
check_volume_data(volume1, volume1_data)
def test_statefulset_restore(
client,
core_api,
storage_class, # NOQA
statefulset): # NOQA
"""
Test that data can be restored into volumes usable by a StatefulSet.
"""
statefulset_name = 'statefulset-restore-test'
update_test_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
create_and_test_backups(core_api, client, pod_info)
delete_and_wait_statefulset(core_api, client, statefulset)
csi = True
try:
core_api.read_namespaced_pod(name='csi-provisioner-0',
namespace='longhorn-system')
except ApiException as e:
if (e.status == 404):
csi = False
# StatefulSet fixture already cleans these up, use the manifests instead of
# the fixtures to avoid issues during teardown.
pv = {
'apiVersion': 'v1',
'kind': 'PersistentVolume',
'metadata': {
'name': ''
},
'spec': {
'capacity': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
},
'volumeMode': 'Filesystem',
'accessModes': ['ReadWriteOnce'],
'persistentVolumeReclaimPolicy': 'Delete',
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
pvc = {
'apiVersion': 'v1',
'kind': 'PersistentVolumeClaim',
'metadata': {
'name': ''
},
'spec': {
'accessModes': ['ReadWriteOnce'],
'resources': {
'requests': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
},
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
if csi:
pv['spec']['csi'] = {
'driver': 'io.rancher.longhorn',
'fsType': 'ext4',
'volumeAttributes': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout']
},
'volumeHandle': ''
}
else:
pv['spec']['flexVolume'] = {
'driver': 'rancher.io/longhorn',
'fsType': 'ext4',
'options': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout'],
'fromBackup':
'',
'size':
size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
}
# Make sure that volumes still work even if the Pod and StatefulSet names
# are different.
for pod in pod_info:
pod['pod_name'] = pod['pod_name'].replace(
'statefulset-restore-test', 'statefulset-restore-test-2')
pod['pvc_name'] = pod['pvc_name'].replace(
'statefulset-restore-test', 'statefulset-restore-test-2')
pv['metadata']['name'] = pod['pvc_name']
if csi:
client.create_volume(
name=pod['pvc_name'],
size=size_to_string(DEFAULT_VOLUME_SIZE * Gi),
numberOfReplicas=int(
storage_class['parameters']['numberOfReplicas']),
fromBackup=pod['backup_snapshot']['url'])
wait_for_volume_detached(client, pod['pvc_name'])
pv['spec']['csi']['volumeHandle'] = pod['pvc_name']
else:
pv['spec']['flexVolume']['options']['fromBackup'] = \
pod['backup_snapshot']['url']
core_api.create_persistent_volume(pv)
pvc['metadata']['name'] = pod['pvc_name']
pvc['spec']['volumeName'] = pod['pvc_name']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
statefulset_name = 'statefulset-restore-test-2'
update_test_manifests(statefulset, storage_class, statefulset_name)
create_and_wait_statefulset(statefulset)
for pod in pod_info:
resp = read_volume_data(core_api, pod['pod_name'])
assert resp == pod['data']
def test_kubernetes_status(
client,
core_api,
storage_class, # NOQA
statefulset,
csi_pv,
pvc,
pod): # NOQA
statefulset_name = 'kubernetes-status-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
storage_class['reclaimPolicy'] = 'Retain'
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
volume_info = [p['pv_name'] for p in pod_info]
extra_pod_name = 'extra-pod-using-' + volume_info[1]
pod['metadata']['name'] = extra_pod_name
p2 = core_api.read_namespaced_pod(name=pod_info[1]['pod_name'],
namespace='default')
pod['spec']['nodeName'] = p2.spec.node_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pod_info[1]['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == p['pv_name']
assert k_status['pvStatus'] == 'Bound'
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert not k_status['lastPVCRefAt']
assert not k_status['lastPodRefAt']
if i == 0:
assert len(workloads) == 1
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert workloads[0]['podStatus'] == 'Running'
if i == 1:
assert len(k_status['workloadsStatus']) == 2
if workloads[0]['podName'] == pod_info[i]['pod_name']:
assert workloads[1]['podName'] == extra_pod_name
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
assert not workloads[1]['workloadName']
assert not workloads[1]['workloadType']
else:
assert workloads[1]['podName'] == pod_info[i]['pod_name']
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
assert workloads[1]['workloadName'] == statefulset_name
assert workloads[1]['workloadType'] == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running' and \
workloads[1]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 2
assert workloads[0]['podStatus'] == 'Running'
assert workloads[1]['podStatus'] == 'Running'
ks_list = [{}, {}]
delete_and_wait_statefulset_only(core_api, statefulset)
# the extra pod is still using the 2nd volume
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
ks_list[i]['pvName'] = p['pv_name']
ks_list[i]['pvStatus'] = 'Bound'
ks_list[i]['namespace'] = 'default'
ks_list[i]['pvcName'] = p['pvc_name']
ks_list[i]['lastPVCRefAt'] = ''
if i == 0:
ks_list[i]['lastPodRefAt'] = 'not empty'
ks_list[i]['workloadsStatus'] = [
{
'podName': p['pod_name'],
'podStatus': 'Running',
'workloadName': statefulset_name,
'workloadType': 'StatefulSet',
},
]
if i == 1:
ks_list[i]['lastPodRefAt'] = ''
ks_list[i]['workloadsStatus'] = [{
'podName': extra_pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
}]
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted extra_pod, all volumes have no workload
delete_and_wait_pod(core_api, pod['metadata']['name'])
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
ks_list[i]['lastPodRefAt'] = 'not empty'
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted pvc only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pvc(core_api, p['pvc_name'])
ks_list[i]['pvStatus'] = 'Released'
ks_list[i]['lastPVCRefAt'] = 'not empty'
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted pv only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pv(core_api, p['pv_name'])
ks_list[i]['pvName'] = ''
ks_list[i]['pvStatus'] = ''
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# reuse that volume
for p, volume_name in zip(pod_info, volume_info):
p['pod_name'] = p['pod_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pvc_name'] = p['pvc_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pv_name'] = p['pvc_name']
csi_pv['metadata']['name'] = p['pv_name']
csi_pv['spec']['csi']['volumeHandle'] = volume_name
core_api.create_persistent_volume(csi_pv)
pvc['metadata']['name'] = p['pvc_name']
pvc['spec']['volumeName'] = p['pv_name']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
pod['metadata']['name'] = p['pod_name']
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': p['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'pvName':
p['pv_name'],
'pvStatus':
'Bound',
'namespace':
'default',
'pvcName':
p['pvc_name'],
'lastPVCRefAt':
'',
'lastPodRefAt':
'',
'workloadsStatus': [
{
'podName': p['pod_name'],
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
},
],
}
wait_volume_kubernetes_status(client, volume_name, ks)
delete_and_wait_pod(core_api, p['pod_name'])
# Since persistentVolumeReclaimPolicy of csi_pv is `Delete`,
# we don't need to delete bounded pv manually
delete_and_wait_pvc(core_api, p['pvc_name'])
wait_delete_pv(core_api, p['pv_name'])