def restore_csi_volume_snapshot(core_api, client, csivolsnap, pvc_name,
pvc_request_storage_size): # NOQA
restore_pvc = {
'apiVersion': 'v1',
'kind': 'PersistentVolumeClaim',
'metadata': {
'name': pvc_name
},
'spec': {
'accessModes': ['ReadWriteOnce'],
'resources': {
'requests': {
'storage': pvc_request_storage_size
}
},
'storageClassName': 'longhorn',
'dataSource': {
'kind': 'VolumeSnapshot',
'apiGroup': 'snapshot.storage.k8s.io',
'name': csivolsnap["metadata"]["name"]
}
}
}
core_api.create_namespaced_persistent_volume_claim(body=restore_pvc,
namespace='default')
restore_volume_name = None
restore_pvc_name = restore_pvc["metadata"]["name"]
for i in range(RETRY_COUNTS):
restore_pvc = \
core_api.read_namespaced_persistent_volume_claim(
name=restore_pvc_name,
namespace="default")
if restore_pvc.spec.volume_name is not None:
restore_volume_name = restore_pvc.spec.volume_name
break
time.sleep(RETRY_INTERVAL)
assert restore_volume_name is not None
wait_for_volume_restoration_completed(client, restore_volume_name)
wait_for_volume_detached(client, restore_volume_name)
return restore_pvc
def test_kubernetes_status(
client,
core_api,
storage_class, # NOQA
statefulset,
csi_pv,
pvc,
pod): # NOQA
"""
Test Volume feature: Kubernetes Status
1. Create StorageClass with `reclaimPolicy = Retain`
2. Create a statefulset `kubernetes-status-test` with the StorageClass
1. The statefulset has scale of 2.
3. Get the volume name from the SECOND pod of the StateufulSet pod and
create an `extra_pod` with the same volume on the same node
4. Check the volumes that used by the StatefulSet
1. The volume used by the FIRST StatefulSet pod will have one workload
2. The volume used by the SECOND StatefulSet pod will have two
workloads
3. Validate related status, e.g. pv/pod name/state, workload
name/type
5. Check the volumes again
1. PV/PVC should still be bound
2. The volume used by the FIRST pod should have history data
3. The volume used by the SECOND and extra pod should have current data
point to the extra pod
6. Delete the extra pod
1. Now all the volume's should only have history data(`lastPodRefAt`
set)
7. Delete the PVC
1. PVC should be updated with status `Released` and become history data
8. Delete PV
1. All the Kubernetes status information should be cleaned up.
9. Reuse the two Longhorn volumes to create new pods
1. Since the `reclaimPolicy == Retain`, volume won't be deleted by
Longhorn
2. Check the Kubernetes status now updated, with pod info but empty
workload
3. Default Longhorn Static StorageClass will remove the PV with PVC,
but leave Longhorn volume
"""
statefulset_name = 'kubernetes-status-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
storage_class['reclaimPolicy'] = 'Retain'
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
volume_info = [p['pv_name'] for p in pod_info]
extra_pod_name = 'extra-pod-using-' + volume_info[1]
pod['metadata']['name'] = extra_pod_name
p2 = core_api.read_namespaced_pod(name=pod_info[1]['pod_name'],
namespace='default')
pod['spec']['nodeName'] = p2.spec.node_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pod_info[1]['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i] # NOQA
volume = client.by_id_volume(volume_name)
k_status = volume.kubernetesStatus
workloads = k_status.workloadsStatus
assert k_status.pvName == p['pv_name']
assert k_status.pvStatus == 'Bound'
assert k_status.namespace == 'default'
assert k_status.pvcName == p['pvc_name']
assert not k_status.lastPVCRefAt
assert not k_status.lastPodRefAt
if i == 0:
assert len(workloads) == 1
assert workloads[0].podName == p['pod_name']
assert workloads[0].workloadName == statefulset_name
assert workloads[0].workloadType == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0].podStatus == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume.kubernetesStatus
workloads = k_status.workloadsStatus
assert workloads[0].podStatus == 'Running'
if i == 1:
assert len(k_status.workloadsStatus) == 2
if workloads[0].podName == pod_info[i]['pod_name']:
assert workloads[1].podName == extra_pod_name
assert workloads[0].workloadName == statefulset_name
assert workloads[0].workloadType == 'StatefulSet'
assert not workloads[1].workloadName
assert not workloads[1].workloadType
else:
assert workloads[1].podName == pod_info[i]['pod_name']
assert workloads[0].podName == extra_pod_name
assert not workloads[0].workloadName
assert not workloads[0].workloadType
assert workloads[1].workloadName == statefulset_name
assert workloads[1].workloadType == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0].podStatus == 'Running' and \
workloads[1].podStatus == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume.kubernetesStatus
workloads = k_status.workloadsStatus
assert len(workloads) == 2
assert workloads[0].podStatus == 'Running'
assert workloads[1].podStatus == 'Running'
ks_list = [{}, {}]
delete_and_wait_statefulset_only(core_api, statefulset)
# the extra pod is still using the 2nd volume
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
ks_list[i]['pvName'] = p['pv_name']
ks_list[i]['pvStatus'] = 'Bound'
ks_list[i]['namespace'] = 'default'
ks_list[i]['pvcName'] = p['pvc_name']
ks_list[i]['lastPVCRefAt'] = ''
if i == 0:
ks_list[i]['lastPodRefAt'] = 'not empty'
ks_list[i]['workloadsStatus'] = [
{
'podName': p['pod_name'],
'podStatus': 'Running',
'workloadName': statefulset_name,
'workloadType': 'StatefulSet',
},
]
if i == 1:
ks_list[i]['lastPodRefAt'] = ''
ks_list[i]['workloadsStatus'] = [{
'podName': extra_pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
}]
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted extra_pod, all volumes have no workload
delete_and_wait_pod(core_api, pod['metadata']['name'])
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
ks_list[i]['lastPodRefAt'] = 'not empty'
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted pvc only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pvc(core_api, p['pvc_name'])
ks_list[i]['pvStatus'] = 'Released'
ks_list[i]['lastPVCRefAt'] = 'not empty'
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted pv only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pv(core_api, p['pv_name'])
ks_list[i]['pvName'] = ''
ks_list[i]['pvStatus'] = ''
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# reuse that volume
for p, volume_name in zip(pod_info, volume_info):
p['pod_name'] = p['pod_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pvc_name'] = p['pvc_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pv_name'] = p['pvc_name']
csi_pv['metadata']['name'] = p['pv_name']
csi_pv['spec']['csi']['volumeHandle'] = volume_name
csi_pv['spec']['storageClassName'] = \
DEFAULT_LONGHORN_STATIC_STORAGECLASS_NAME
core_api.create_persistent_volume(csi_pv)
pvc['metadata']['name'] = p['pvc_name']
pvc['spec']['volumeName'] = p['pv_name']
pvc['spec']['storageClassName'] = \
DEFAULT_LONGHORN_STATIC_STORAGECLASS_NAME
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
pod['metadata']['name'] = p['pod_name']
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': p['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'pvName':
p['pv_name'],
'pvStatus':
'Bound',
'namespace':
'default',
'pvcName':
p['pvc_name'],
'lastPVCRefAt':
'',
'lastPodRefAt':
'',
'workloadsStatus': [
{
'podName': p['pod_name'],
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
},
],
}
wait_volume_kubernetes_status(client, volume_name, ks)
delete_and_wait_pod(core_api, p['pod_name'])
# Since persistentVolumeReclaimPolicy of csi_pv is `Delete`,
# we don't need to delete bounded pv manually
delete_and_wait_pvc(core_api, p['pvc_name'])
wait_delete_pv(core_api, p['pv_name'])
def test_csi_minimal_volume_size(
client, core_api, csi_pv, pvc, pod_make): # NOQA
"""
Test CSI Minimal Volume Size
1. Create a PVC requesting size 5MiB. Check the PVC requested size is
5MiB and capacity size get is 10MiB.
2. Remove the PVC.
3. Create a PVC requesting size 10MiB. Check the PVC requested size and
capacity size get are both 10MiB.
4. Create a pod to use this PVC.
5. Write some data to the volume and read it back to compare.
"""
vol_name = generate_volume_name()
create_and_check_volume(client, vol_name, size=str(100*Mi))
low_storage = str(5*Mi)
min_storage = str(10*Mi)
pv_name = vol_name + "-pv"
csi_pv['metadata']['name'] = pv_name
csi_pv['spec']['csi']['volumeHandle'] = vol_name
csi_pv['spec']['capacity']['storage'] = min_storage
core_api.create_persistent_volume(csi_pv)
pvc_name = vol_name + "-pvc"
pvc['metadata']['name'] = pvc_name
pvc['spec']['volumeName'] = pv_name
pvc['spec']['resources']['requests']['storage'] = low_storage
pvc['spec']['storageClassName'] = ''
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
claim = common.wait_for_pvc_phase(core_api, pvc_name, "Bound")
assert claim.spec.resources.requests['storage'] == low_storage
assert claim.status.capacity['storage'] == min_storage
common.delete_and_wait_pvc(core_api, pvc_name)
common.delete_and_wait_pv(core_api, pv_name)
wait_for_volume_detached(client, vol_name)
core_api.create_persistent_volume(csi_pv)
pvc['spec']['resources']['requests']['storage'] = min_storage
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
claim = common.wait_for_pvc_phase(core_api, pvc_name, "Bound")
assert claim.spec.resources.requests['storage'] == min_storage
assert claim.status.capacity['storage'] == min_storage
pod_name = vol_name + '-pod'
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
create_and_wait_pod(core_api, pod)
test_data = "longhorn-integration-test"
test_file = "test"
write_pod_volume_data(core_api, pod_name, test_data, test_file)
read_data = read_volume_data(core_api, pod_name, test_file)
assert read_data == test_data
def test_statefulset_restore(
client,
core_api,
storage_class, # NOQA
statefulset): # NOQA
"""
Test that data can be restored into volumes usable by a StatefulSet.
1. Create a StatefulSet with VolumeClaimTemplate and Longhorn.
2. Wait for pods to run.
3. Create a backup for each pod.
4. Delete the StatefulSet, including the Longhorn volumes.
5. Create volumes and PV/PVC using previous backups from each Pod.
1. PVs will be created using the previous names.
2. PVCs will be created using previous name + "-2" due to statefulset
has a naming policy for what should be PVC name for them.
6. Create a new StatefulSet using the previous name + "-2"
7. Wait for pods to be up.
. Verify the pods contain the previous backed up data
"""
statefulset_name = 'statefulset-restore-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
create_and_test_backups(core_api, client, pod_info)
delete_and_wait_statefulset(core_api, client, statefulset)
csi = check_csi(core_api)
# StatefulSet fixture already cleans these up, use the manifests instead of
# the fixtures to avoid issues during teardown.
pv = {
'apiVersion': 'v1',
'kind': 'PersistentVolume',
'metadata': {
'name': ''
},
'spec': {
'capacity': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
},
'volumeMode': 'Filesystem',
'accessModes': ['ReadWriteOnce'],
'persistentVolumeReclaimPolicy': 'Delete',
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
pvc = {
'apiVersion': 'v1',
'kind': 'PersistentVolumeClaim',
'metadata': {
'name': ''
},
'spec': {
'accessModes': ['ReadWriteOnce'],
'resources': {
'requests': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
},
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
assert csi
pv['spec']['csi'] = {
'driver': 'driver.longhorn.io',
'fsType': 'ext4',
'volumeAttributes': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout']
},
'volumeHandle': ''
}
# Make sure that volumes still work even if the Pod and StatefulSet names
# are different.
for pod in pod_info:
pod['pod_name'] = pod['pod_name'].replace(
'statefulset-restore-test', 'statefulset-restore-test-2')
pod['pvc_name'] = pod['pvc_name'].replace(
'statefulset-restore-test', 'statefulset-restore-test-2')
pv['metadata']['name'] = pod['pvc_name']
client.create_volume(
name=pod['pvc_name'],
size=size_to_string(DEFAULT_VOLUME_SIZE * Gi),
numberOfReplicas=int(
storage_class['parameters']['numberOfReplicas']),
fromBackup=pod['backup_snapshot']['url'])
wait_for_volume_detached(client, pod['pvc_name'])
pv['spec']['csi']['volumeHandle'] = pod['pvc_name']
core_api.create_persistent_volume(pv)
pvc['metadata']['name'] = pod['pvc_name']
pvc['spec']['volumeName'] = pod['pvc_name']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
statefulset_name = 'statefulset-restore-test-2'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_and_wait_statefulset(statefulset)
for pod in pod_info:
resp = read_volume_data(core_api, pod['pod_name'])
assert resp == pod['data']
def test_rwx_multi_statefulset_with_same_pvc(core_api, pvc, statefulset,
pod): # NOQA
"""
Test writing of data into a volume from multiple pods using same PVC
1. Create a volume with 'accessMode' rwx.
2. Create a PV and a PVC with access mode 'readwritemany' and attach to the
volume.
3. Deploy a StatefulSet of 2 pods with the existing PVC above created.
4. Wait for both pods to come up.
5. Create a pod with the existing PVC above created.
6. Wait for StatefulSet to come up healthy.
7. Write data all three pods and compute md5sum.
8. Check the data md5sum in the share manager pod.
"""
pvc_name = 'pvc-multi-pods-test'
statefulset_name = 'statefulset-rwx-same-pvc-test'
pod_name = 'pod-rwx-same-pvc-test'
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = 'longhorn'
pvc['spec']['accessModes'] = ['ReadWriteMany']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
statefulset['metadata']['name'] = \
statefulset['spec']['selector']['matchLabels']['app'] = \
statefulset['spec']['serviceName'] = \
statefulset['spec']['template']['metadata']['labels']['app'] = \
statefulset_name
statefulset['spec']['template']['spec']['volumes'] = \
[create_pvc_spec(pvc_name)]
del statefulset['spec']['volumeClaimTemplates']
create_and_wait_statefulset(statefulset)
pv_name = get_volume_name(core_api, pvc_name)
share_manager_name = 'share-manager-' + pv_name
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api,
statefulset_name + '-0',
test_data,
filename='test1')
assert test_data == read_volume_data(core_api,
statefulset_name + '-1',
filename='test1')
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
create_and_wait_pod(core_api, pod)
assert test_data == read_volume_data(core_api, pod_name, filename='test1')
test_data_2 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data_2, filename='test2')
command1 = 'cat /export' + '/' + pv_name + '/' + 'test1'
command2 = 'cat /export' + '/' + pv_name + '/' + 'test2'
assert test_data == exec_command_in_pod(core_api, command1,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_2 == exec_command_in_pod(core_api, command2,
share_manager_name,
LONGHORN_NAMESPACE)
def test_rwx_deployment_with_multi_pods(core_api, pvc,
make_deployment_with_pvc): # NOQA
"""
Test deployment of 2 pods with same PVC.
1. Create a volume with 'accessMode' rwx.
2. Create a PV and a PVC with access mode 'readwritemany' and attach to the
volume.
3. Create a deployment of 2 pods with PVC created
4. Wait for 2 pods to come up healthy.
5. Write data in both pods and compute md5sum.
6. Check the data md5sum in the share manager pod.
"""
pvc_name = 'pvc-deployment-multi-pods-test'
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = 'longhorn'
pvc['spec']['accessModes'] = ['ReadWriteMany']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
deployment = make_deployment_with_pvc('deployment-multi-pods-test',
pvc_name,
replicas=2)
apps_api = get_apps_api_client()
create_and_wait_deployment(apps_api, deployment)
pv_name = get_volume_name(core_api, pvc_name)
share_manager_name = 'share-manager-' + pv_name
deployment_label_selector = "name=" + \
deployment["metadata"]["labels"]["name"]
deployment_pod_list = \
core_api.list_namespaced_pod(namespace="default",
label_selector=deployment_label_selector)
assert deployment_pod_list.items.__len__() == 2
pod_name_1 = deployment_pod_list.items[0].metadata.name
test_data_1 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name_1, test_data_1, filename='test1')
pod_name_2 = deployment_pod_list.items[1].metadata.name
command = 'cat /data/test1'
pod_data_2 = exec_command_in_pod(core_api, command, pod_name_2, 'default')
assert test_data_1 == pod_data_2
test_data_2 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name_2, test_data_2, filename='test2')
command = 'cat /export' + '/' + pv_name + '/' + 'test1'
share_manager_data_1 = exec_command_in_pod(core_api, command,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_1 == share_manager_data_1
command = 'cat /export' + '/' + pv_name + '/' + 'test2'
share_manager_data_2 = exec_command_in_pod(core_api, command,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_2 == share_manager_data_2
def test_kubernetes_status(client, core_api, storage_class, # NOQA
statefulset, csi_pv, pvc, pod): # NOQA
statefulset_name = 'kubernetes-status-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
storage_class['reclaimPolicy'] = 'Retain'
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
volume_info = [p['pv_name'] for p in pod_info]
extra_pod_name = 'extra-pod-using-' + volume_info[1]
pod['metadata']['name'] = extra_pod_name
p2 = core_api.read_namespaced_pod(name=pod_info[1]['pod_name'],
namespace='default')
pod['spec']['nodeName'] = p2.spec.node_name
pod['spec']['volumes'] = [{
'name': pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pod_info[1]['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == p['pv_name']
assert k_status['pvStatus'] == 'Bound'
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert not k_status['lastPVCRefAt']
assert not k_status['lastPodRefAt']
if i == 0:
assert len(workloads) == 1
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert workloads[0]['podStatus'] == 'Running'
if i == 1:
assert len(k_status['workloadsStatus']) == 2
if workloads[0]['podName'] == pod_info[i]['pod_name']:
assert workloads[1]['podName'] == extra_pod_name
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
assert not workloads[1]['workloadName']
assert not workloads[1]['workloadType']
else:
assert workloads[1]['podName'] == pod_info[i]['pod_name']
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
assert workloads[1]['workloadName'] == statefulset_name
assert workloads[1]['workloadType'] == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running' and \
workloads[1]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 2
assert workloads[0]['podStatus'] == 'Running'
assert workloads[1]['podStatus'] == 'Running'
# the extra pod is still using the 2nd volume
delete_and_wait_statefulset_only(core_api, statefulset)
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == p['pv_name']
assert k_status['pvStatus'] == 'Bound'
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert not k_status['lastPVCRefAt']
assert len(workloads) == 1
if i == 0:
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
assert k_status['lastPodRefAt']
if i == 1:
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
assert not k_status['lastPodRefAt']
# deleted extra_pod, all volumes have no workload
delete_and_wait_pod(core_api, pod['metadata']['name'])
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == p['pv_name']
assert k_status['pvStatus'] == 'Bound'
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert not k_status['lastPVCRefAt']
assert k_status['lastPodRefAt']
assert len(workloads) == 1
if i == 0:
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
if i == 1:
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
# deleted pvc only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pvc(core_api, p['pvc_name'])
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
for _ in range(RETRY_COUNTS):
if k_status['pvStatus'] == 'Released':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == p['pv_name']
assert k_status['pvStatus'] == 'Released'
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert k_status['lastPVCRefAt']
assert k_status['lastPodRefAt']
assert len(workloads) == 1
if i == 0:
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
if i == 1:
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
# deleted pv only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pv(core_api, p['pv_name'])
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == ''
assert k_status['pvStatus'] == ''
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert k_status['lastPVCRefAt']
assert k_status['lastPodRefAt']
assert len(workloads) == 1
if i == 0:
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
if i == 1:
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
# reuse that volume
for p, volume_name in zip(pod_info, volume_info):
p['pod_name'] = p['pod_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pvc_name'] = p['pvc_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pv_name'] = p['pvc_name']
csi_pv['metadata']['name'] = p['pv_name']
csi_pv['spec']['csi']['volumeHandle'] = volume_name
core_api.create_persistent_volume(csi_pv)
pvc['metadata']['name'] = p['pvc_name']
pvc['spec']['volumeName'] = p['pv_name']
core_api.create_namespaced_persistent_volume_claim(
body=pvc, namespace='default')
pod['metadata']['name'] = p['pod_name']
pod['spec']['volumes'] = [{
'name': pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': p['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 1
assert k_status['pvName'] == p['pv_name']
for _ in range(RETRY_COUNTS):
if k_status['pvStatus'] == 'Bound':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 1
assert k_status['pvStatus'] == 'Bound'
for _ in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 1
assert workloads[0]['podStatus'] == 'Running'
assert workloads[0]['podName'] == p['pod_name']
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert not k_status['lastPVCRefAt']
assert not k_status['lastPodRefAt']
delete_and_wait_pod(core_api, p['pod_name'])
# Since persistentVolumeReclaimPolicy of csi_pv is `Delete`,
# we don't need to delete bounded pv manually
delete_and_wait_pvc(core_api, p['pvc_name'])
wait_delete_pv(core_api, p['pv_name'])
def test_kubernetes_status(
client,
core_api,
storage_class, # NOQA
statefulset,
csi_pv,
pvc,
pod): # NOQA
statefulset_name = 'kubernetes-status-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
storage_class['reclaimPolicy'] = 'Retain'
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
volume_info = [p['pv_name'] for p in pod_info]
extra_pod_name = 'extra-pod-using-' + volume_info[1]
pod['metadata']['name'] = extra_pod_name
p2 = core_api.read_namespaced_pod(name=pod_info[1]['pod_name'],
namespace='default')
pod['spec']['nodeName'] = p2.spec.node_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pod_info[1]['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == p['pv_name']
assert k_status['pvStatus'] == 'Bound'
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == p['pvc_name']
assert not k_status['lastPVCRefAt']
assert not k_status['lastPodRefAt']
if i == 0:
assert len(workloads) == 1
assert workloads[0]['podName'] == p['pod_name']
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert workloads[0]['podStatus'] == 'Running'
if i == 1:
assert len(k_status['workloadsStatus']) == 2
if workloads[0]['podName'] == pod_info[i]['pod_name']:
assert workloads[1]['podName'] == extra_pod_name
assert workloads[0]['workloadName'] == statefulset_name
assert workloads[0]['workloadType'] == 'StatefulSet'
assert not workloads[1]['workloadName']
assert not workloads[1]['workloadType']
else:
assert workloads[1]['podName'] == pod_info[i]['pod_name']
assert workloads[0]['podName'] == extra_pod_name
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
assert workloads[1]['workloadName'] == statefulset_name
assert workloads[1]['workloadType'] == 'StatefulSet'
for _ in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running' and \
workloads[1]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
volume = client.by_id_volume(volume_name)
k_status = volume["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 2
assert workloads[0]['podStatus'] == 'Running'
assert workloads[1]['podStatus'] == 'Running'
ks_list = [{}, {}]
delete_and_wait_statefulset_only(core_api, statefulset)
# the extra pod is still using the 2nd volume
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
ks_list[i]['pvName'] = p['pv_name']
ks_list[i]['pvStatus'] = 'Bound'
ks_list[i]['namespace'] = 'default'
ks_list[i]['pvcName'] = p['pvc_name']
ks_list[i]['lastPVCRefAt'] = ''
if i == 0:
ks_list[i]['lastPodRefAt'] = 'not empty'
ks_list[i]['workloadsStatus'] = [
{
'podName': p['pod_name'],
'podStatus': 'Running',
'workloadName': statefulset_name,
'workloadType': 'StatefulSet',
},
]
if i == 1:
ks_list[i]['lastPodRefAt'] = ''
ks_list[i]['workloadsStatus'] = [{
'podName': extra_pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
}]
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted extra_pod, all volumes have no workload
delete_and_wait_pod(core_api, pod['metadata']['name'])
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
ks_list[i]['lastPodRefAt'] = 'not empty'
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted pvc only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pvc(core_api, p['pvc_name'])
ks_list[i]['pvStatus'] = 'Released'
ks_list[i]['lastPVCRefAt'] = 'not empty'
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# deleted pv only.
for i in range(len(volume_info)):
p, volume_name = pod_info[i], volume_info[i]
delete_and_wait_pv(core_api, p['pv_name'])
ks_list[i]['pvName'] = ''
ks_list[i]['pvStatus'] = ''
wait_volume_kubernetes_status(client, volume_name, ks_list[i])
# reuse that volume
for p, volume_name in zip(pod_info, volume_info):
p['pod_name'] = p['pod_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pvc_name'] = p['pvc_name'].replace('kubernetes-status-test',
'kubernetes-status-test-reuse')
p['pv_name'] = p['pvc_name']
csi_pv['metadata']['name'] = p['pv_name']
csi_pv['spec']['csi']['volumeHandle'] = volume_name
core_api.create_persistent_volume(csi_pv)
pvc['metadata']['name'] = p['pvc_name']
pvc['spec']['volumeName'] = p['pv_name']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
pod['metadata']['name'] = p['pod_name']
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': p['pvc_name'],
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'pvName':
p['pv_name'],
'pvStatus':
'Bound',
'namespace':
'default',
'pvcName':
p['pvc_name'],
'lastPVCRefAt':
'',
'lastPodRefAt':
'',
'workloadsStatus': [
{
'podName': p['pod_name'],
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
},
],
}
wait_volume_kubernetes_status(client, volume_name, ks)
delete_and_wait_pod(core_api, p['pod_name'])
# Since persistentVolumeReclaimPolicy of csi_pv is `Delete`,
# we don't need to delete bounded pv manually
delete_and_wait_pvc(core_api, p['pvc_name'])
wait_delete_pv(core_api, p['pv_name'])
def test_statefulset_restore(
client,
core_api,
storage_class, # NOQA
statefulset): # NOQA
"""
Test that data can be restored into volumes usable by a StatefulSet.
"""
statefulset_name = 'statefulset-restore-test'
update_test_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
create_and_test_backups(core_api, client, pod_info)
delete_and_wait_statefulset(core_api, client, statefulset)
csi = True
try:
core_api.read_namespaced_pod(name='csi-provisioner-0',
namespace='longhorn-system')
except ApiException as e:
if (e.status == 404):
csi = False
# StatefulSet fixture already cleans these up, use the manifests instead of
# the fixtures to avoid issues during teardown.
pv = {
'apiVersion': 'v1',
'kind': 'PersistentVolume',
'metadata': {
'name': ''
},
'spec': {
'capacity': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
},
'volumeMode': 'Filesystem',
'accessModes': ['ReadWriteOnce'],
'persistentVolumeReclaimPolicy': 'Delete',
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
pvc = {
'apiVersion': 'v1',
'kind': 'PersistentVolumeClaim',
'metadata': {
'name': ''
},
'spec': {
'accessModes': ['ReadWriteOnce'],
'resources': {
'requests': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
},
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
if csi:
pv['spec']['csi'] = {
'driver': 'io.rancher.longhorn',
'fsType': 'ext4',
'volumeAttributes': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout']
},
'volumeHandle': ''
}
else:
pv['spec']['flexVolume'] = {
'driver': 'rancher.io/longhorn',
'fsType': 'ext4',
'options': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout'],
'fromBackup':
'',
'size':
size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
}
# Make sure that volumes still work even if the Pod and StatefulSet names
# are different.
for pod in pod_info:
pod['pod_name'] = pod['pod_name'].replace(
'statefulset-restore-test', 'statefulset-restore-test-2')
pod['pvc_name'] = pod['pvc_name'].replace(
'statefulset-restore-test', 'statefulset-restore-test-2')
pv['metadata']['name'] = pod['pvc_name']
if csi:
client.create_volume(
name=pod['pvc_name'],
size=size_to_string(DEFAULT_VOLUME_SIZE * Gi),
numberOfReplicas=int(
storage_class['parameters']['numberOfReplicas']),
fromBackup=pod['backup_snapshot']['url'])
wait_for_volume_detached(client, pod['pvc_name'])
pv['spec']['csi']['volumeHandle'] = pod['pvc_name']
else:
pv['spec']['flexVolume']['options']['fromBackup'] = \
pod['backup_snapshot']['url']
core_api.create_persistent_volume(pv)
pvc['metadata']['name'] = pod['pvc_name']
pvc['spec']['volumeName'] = pod['pvc_name']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
statefulset_name = 'statefulset-restore-test-2'
update_test_manifests(statefulset, storage_class, statefulset_name)
create_and_wait_statefulset(statefulset)
for pod in pod_info:
resp = read_volume_data(core_api, pod['pod_name'])
assert resp == pod['data']
def test_recurring_jobs_when_volume_detached_unexpectedly(
settings_reset, set_random_backupstore, client, core_api, apps_api,
pvc, make_deployment_with_pvc): # NOQA
"""
Test recurring jobs when volume detached unexpectedly
Context:
If the volume is automatically attached by the recurring backup job,
make sure that workload pod eventually is able to use the volume
when volume is detached unexpectedly during the backup process.
Steps:
1. Create a volume, attach to a pod of a deployment,
write 500MB to the volume.
2. Scale down the deployment. The volume is detached.
3. Turn on `Allow Recurring Job While Volume Is Detached` setting.
4. Create a recurring backup job that runs every 2 mins.
5. Wait until the recurring backup job starts and the backup progress
is > 50%, kill the engine process of the volume.
6. Verify volume automatically reattached and is healthy again.
7. Wait until the backup finishes.
8. Wait for the volume to be in detached state with
`frontendDisabled=false`
9. Scale up the deployment.
Verify that we can read the file `lost+found` from the workload pod
10. Turn off `Allow Recurring Job While Volume Is Detached` setting
Clean up backups, volumes.
"""
recurring_job_setting = \
client.by_id_setting(SETTING_RECURRING_JOB_WHILE_VOLUME_DETACHED)
client.update(recurring_job_setting, value="true")
pvc_name = 'pvc-volume-detached-unexpectedly-test'
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = 'longhorn'
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
deployment = make_deployment_with_pvc(
'deployment-volume-detached-unexpectedly-test', pvc_name)
create_and_wait_deployment(apps_api, deployment)
pod_names = common.get_deployment_pod_names(core_api, deployment)
vol_name = get_volume_name(core_api, pvc_name)
write_pod_volume_random_data(core_api, pod_names[0], "/data/test",
DATA_SIZE_IN_MB_3)
data = read_volume_data(core_api, pod_names[0], 'default')
deployment['spec']['replicas'] = 0
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment["metadata"]["name"])
vol = wait_for_volume_detached(client, vol_name)
jobs = [{
"name": RECURRING_JOB_NAME,
"cron": "*/2 * * * *",
"task": "backup",
"retain": 1
}]
vol.recurringUpdate(jobs=jobs)
time.sleep(60)
wait_for_recurring_backup_to_start(client,
core_api,
vol_name,
expected_snapshot_count=1,
minimum_progress=50)
crash_engine_process_with_sigkill(client, core_api, vol_name)
# Check if the volume is reattached after recurring backup is interrupted
time.sleep(10)
wait_for_volume_healthy_no_frontend(client, vol_name)
# Since the backup state is removed after the backup complete and it
# could happen quickly. Checking for the both in-progress and complete
# state could be hard to catch, thus we only check the complete state
def backup_complete_predicate(b):
return b.state == "complete" and b.error == ""
common.wait_for_backup_state(client, vol_name, backup_complete_predicate)
wait_for_volume_detached(client, vol_name)
deployment['spec']['replicas'] = 1
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment["metadata"]["name"])
wait_deployment_replica_ready(apps_api, deployment["metadata"]["name"], 1)
pod_names = common.get_deployment_pod_names(core_api, deployment)
assert read_volume_data(core_api, pod_names[0], 'default') == data
# Use fixture to cleanup the backupstore and since we
# crashed the engine replica initiated the backup, it's
# backupstore lock will still be present, so we need
# to wait till the lock is expired, before we can delete
# the backups
vol.recurringUpdate(jobs=[])
backupstore.backupstore_wait_for_lock_expiration()
def test_statefulset_restore(client, core_api, storage_class, # NOQA
statefulset): # NOQA
"""
Test that data can be restored into volumes usable by a StatefulSet.
"""
statefulset_name = 'statefulset-restore-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
create_and_test_backups(core_api, client, pod_info)
delete_and_wait_statefulset(core_api, client, statefulset)
csi = check_csi(core_api)
# StatefulSet fixture already cleans these up, use the manifests instead of
# the fixtures to avoid issues during teardown.
pv = {
'apiVersion': 'v1',
'kind': 'PersistentVolume',
'metadata': {
'name': ''
},
'spec': {
'capacity': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
},
'volumeMode': 'Filesystem',
'accessModes': ['ReadWriteOnce'],
'persistentVolumeReclaimPolicy': 'Delete',
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
pvc = {
'apiVersion': 'v1',
'kind': 'PersistentVolumeClaim',
'metadata': {
'name': ''
},
'spec': {
'accessModes': [
'ReadWriteOnce'
],
'resources': {
'requests': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
},
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
if csi:
pv['spec']['csi'] = {
'driver': 'io.rancher.longhorn',
'fsType': 'ext4',
'volumeAttributes': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout']
},
'volumeHandle': ''
}
else:
pv['spec']['flexVolume'] = {
'driver': 'rancher.io/longhorn',
'fsType': 'ext4',
'options': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout'],
'fromBackup': '',
'size': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
}
# Make sure that volumes still work even if the Pod and StatefulSet names
# are different.
for pod in pod_info:
pod['pod_name'] = pod['pod_name'].replace('statefulset-restore-test',
'statefulset-restore-test-2')
pod['pvc_name'] = pod['pvc_name'].replace('statefulset-restore-test',
'statefulset-restore-test-2')
pv['metadata']['name'] = pod['pvc_name']
if csi:
client.create_volume(
name=pod['pvc_name'],
size=size_to_string(DEFAULT_VOLUME_SIZE * Gi),
numberOfReplicas=int(
storage_class['parameters']['numberOfReplicas']),
fromBackup=pod['backup_snapshot']['url'])
wait_for_volume_detached(client, pod['pvc_name'])
pv['spec']['csi']['volumeHandle'] = pod['pvc_name']
else:
pv['spec']['flexVolume']['options']['fromBackup'] = \
pod['backup_snapshot']['url']
core_api.create_persistent_volume(pv)
pvc['metadata']['name'] = pod['pvc_name']
pvc['spec']['volumeName'] = pod['pvc_name']
core_api.create_namespaced_persistent_volume_claim(
body=pvc,
namespace='default')
statefulset_name = 'statefulset-restore-test-2'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_and_wait_statefulset(statefulset)
for pod in pod_info:
resp = read_volume_data(core_api, pod['pod_name'])
assert resp == pod['data']
def test_cloning_basic(client, core_api, pvc, pod, clone_pvc, clone_pod, storage_class_name='longhorn'): # NOQA
"""
1. Create a PVC:
```yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: source-pvc
spec:
storageClassName: longhorn
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 3Gi
```
2. Specify the `source-pvc` in a pod yaml and start the pod
3. Wait for the pod to be running, write some data to the mount
path of the volume
4. Clone a volume by creating the PVC:
```yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cloned-pvc
spec:
storageClassName: longhorn
dataSource:
name: source-pvc
kind: PersistentVolumeClaim
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 3Gi
```
5. Wait for the `CloneStatus.State` in `cloned-pvc` to be `completed`
6. Clone volume should get detached after cloning completion, wait for it.
7. Specify the `cloned-pvc` in a cloned pod yaml and deploy the cloned pod
8. In 3-min retry loop, wait for the cloned pod to be running
9. Verify the data in `cloned-pvc` is the same as in `source-pvc`
10. In 2-min retry loop, verify the volume of the `clone-pvc` eventually
becomes healthy
"""
# Step-1
source_pvc_name = 'source-pvc' + generate_random_suffix()
pvc['metadata']['name'] = source_pvc_name
pvc['spec']['storageClassName'] = storage_class_name
core_api.create_namespaced_persistent_volume_claim(
body=pvc, namespace='default')
wait_for_pvc_phase(core_api, source_pvc_name, "Bound")
# Step-2
pod_name = 'source-pod' + generate_random_suffix()
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(source_pvc_name)]
create_and_wait_pod(core_api, pod)
# Step-3
write_pod_volume_random_data(core_api, pod_name,
'/data/test', DATA_SIZE_IN_MB_2)
source_data = get_pod_data_md5sum(core_api, pod_name, '/data/test')
# Step-4
clone_pvc_name = 'clone-pvc' + generate_random_suffix()
clone_pvc['metadata']['name'] = clone_pvc_name
clone_pvc['spec']['storageClassName'] = storage_class_name
clone_pvc['spec']['dataSource'] = {
'name': source_pvc_name,
'kind': 'PersistentVolumeClaim'
}
core_api.create_namespaced_persistent_volume_claim(
body=clone_pvc, namespace='default')
wait_for_pvc_phase(core_api, clone_pvc_name, "Bound")
# Step-5
clone_volume_name = get_volume_name(core_api, clone_pvc_name)
wait_for_volume_clone_status(client, clone_volume_name, VOLUME_FIELD_STATE,
VOLUME_FIELD_CLONE_COMPLETED)
# Step-6
wait_for_volume_detached(client, clone_volume_name)
# Step-7,8
clone_pod_name = 'clone-pod' + generate_random_suffix()
clone_pod['metadata']['name'] = clone_pod_name
clone_pod['spec']['volumes'] = [create_pvc_spec(clone_pvc_name)]
create_and_wait_pod(core_api, clone_pod)
clone_data = get_pod_data_md5sum(core_api, clone_pod_name, '/data/test')
# Step-9
assert source_data == clone_data
# Step-10
wait_for_volume_healthy(client, clone_volume_name)
def test_cloning_interrupted(client, core_api, pvc, pod, clone_pvc, clone_pod): # NOQA
"""
1. Create a PVC:
```yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: source-pvc
spec:
storageClassName: longhorn
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 3Gi
```
2. Specify the `source-pvc` in a pod yaml and start the pod
3. Wait for the pod to be running, write 500MB of data to the mount
path of the volume
4. Clone a volume by creating the PVC:
```yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cloned-pvc
spec:
storageClassName: longhorn
dataSource:
name: source-pvc
kind: PersistentVolumeClaim
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 3Gi
```
5. Wait for the `CloneStatus.State` in `cloned-pvc` to be `initiated`
6. Kill all replicas process of the `source-pvc`
7. Wait for the `CloneStatus.State` in `cloned-pvc` to be `failed`
8. Clean up `clone-pvc`
9. Redeploy `cloned-pvc` and clone pod
10. In 3-min retry loop, verify cloned pod become running
11. `cloned-pvc` has the same data as `source-pvc`
12. In 2-min retry loop, verify the volume of the `clone-pvc`
eventually becomes healthy.
"""
# Step-1
source_pvc_name = 'source-pvc' + generate_random_suffix()
pvc['metadata']['name'] = source_pvc_name
pvc['spec']['storageClassName'] = 'longhorn'
core_api.create_namespaced_persistent_volume_claim(
body=pvc, namespace='default')
wait_for_pvc_phase(core_api, source_pvc_name, "Bound")
# Step-2
pod_name = 'source-pod' + generate_random_suffix()
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(source_pvc_name)]
create_and_wait_pod(core_api, pod)
# Step-3
write_pod_volume_random_data(core_api, pod_name,
'/data/test', DATA_SIZE_IN_MB_3)
source_data = get_pod_data_md5sum(core_api, pod_name, '/data/test')
source_volume_name = get_volume_name(core_api, source_pvc_name)
# Step-4
clone_pvc_name = 'clone-pvc' + generate_random_suffix()
clone_pvc['metadata']['name'] = clone_pvc_name
clone_pvc['spec']['storageClassName'] = 'longhorn'
clone_pvc['spec']['dataSource'] = {
'name': source_pvc_name,
'kind': 'PersistentVolumeClaim'
}
core_api.create_namespaced_persistent_volume_claim(
body=clone_pvc, namespace='default')
# Step-5
clone_volume_name = get_clone_volume_name(client, source_volume_name)
wait_for_volume_clone_status(client, clone_volume_name, VOLUME_FIELD_STATE,
'initiated')
# Step-6
crash_replica_processes(client, core_api, source_volume_name)
# Step-7
wait_for_volume_faulted(client, source_volume_name)
wait_for_volume_clone_status(client, clone_volume_name, VOLUME_FIELD_STATE,
'failed')
# Step-8
delete_and_wait_pvc(core_api, clone_pvc_name)
# Step-9
clone_pvc_name = 'clone-pvc-2' + generate_random_suffix()
clone_pvc['metadata']['name'] = clone_pvc_name
clone_pvc['spec']['storageClassName'] = 'longhorn'
clone_pvc['spec']['dataSource'] = {
'name': source_pvc_name,
'kind': 'PersistentVolumeClaim'
}
core_api.create_namespaced_persistent_volume_claim(
body=clone_pvc, namespace='default')
wait_for_pvc_phase(core_api, clone_pvc_name, "Bound")
# Step-9
clone_pod_name = 'clone-pod' + generate_random_suffix()
clone_pod['metadata']['name'] = clone_pod_name
clone_pod['spec']['volumes'] = [create_pvc_spec(clone_pvc_name)]
create_and_wait_pod(core_api, clone_pod)
# Step-10
clone_volume_name = get_volume_name(core_api, clone_pvc_name)
wait_for_volume_clone_status(client, clone_volume_name, VOLUME_FIELD_STATE,
VOLUME_FIELD_CLONE_COMPLETED)
# Step-11
clone_data = get_pod_data_md5sum(core_api, clone_pod_name, '/data/test')
assert source_data == clone_data
# Step-12
wait_for_volume_healthy(client, clone_volume_name)
def test_cloning_with_detached_source_volume(client, core_api, pvc, clone_pvc): # NOQA
"""
1. Create a PVC:
```yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: source-pvc
spec:
storageClassName: longhorn
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
```
2. Wait for volume to be created and attach it to a node.
3. Write some data to the mount path of the volume
4. Detach the volume and wait for the volume to be in detached state.
5. Clone a volume by creating the PVC:
```yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cloned-pvc
spec:
storageClassName: longhorn
dataSource:
name: source-pvc
kind: PersistentVolumeClaim
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
```
6. Wait for `source-pvc` to be attached
7. Wait for a new snapshot created in `source-pvc` volume created
8. Wait for the `CloneStatus.State` in `cloned-pvc` to be `completed`
9. Wait for `source-pvc` to be detached
10. Attach the cloned volume to a node
11. Verify the data in `cloned-pvc` is the same as in `source-pvc`.
12. In 2-min retry loop, verify the volume of the `clone-pvc`
eventually becomes healthy.
"""
# Step-1
source_pvc_name = 'source-pvc' + generate_random_suffix()
pvc['metadata']['name'] = source_pvc_name
pvc['spec']['storageClassName'] = 'longhorn'
core_api.create_namespaced_persistent_volume_claim(
body=pvc, namespace='default')
wait_for_pvc_phase(core_api, source_pvc_name, "Bound")
# Step-2
source_volume_name = get_volume_name(core_api, source_pvc_name)
lht_host_id = get_self_host_id()
source_volume = client.by_id_volume(source_volume_name)
source_volume.attach(hostId=lht_host_id)
source_volume = wait_for_volume_healthy(client, source_volume_name)
# Step-3
data = write_volume_random_data(source_volume)
# Steps-4
source_volume.detach(hostId=lht_host_id)
wait_for_volume_detached(client, source_volume_name)
# Step-5
clone_pvc_name = 'clone-pvc' + generate_random_suffix()
clone_pvc['metadata']['name'] = clone_pvc_name
clone_pvc['spec']['storageClassName'] = 'longhorn'
clone_pvc['spec']['dataSource'] = {
'name': source_pvc_name,
'kind': 'PersistentVolumeClaim'
}
core_api.create_namespaced_persistent_volume_claim(
body=clone_pvc, namespace='default')
wait_for_pvc_phase(core_api, clone_pvc_name, "Bound")
# Step-6
source_volume = wait_for_volume_attached(client, source_volume_name)
# Step-7
wait_for_snapshot_count(source_volume, 2)
# Step-8
clone_volume_name = get_volume_name(core_api, clone_pvc_name)
wait_for_volume_clone_status(client, clone_volume_name, VOLUME_FIELD_STATE,
VOLUME_FIELD_CLONE_COMPLETED)
wait_for_volume_detached(client, clone_volume_name)
# Step-9
wait_for_volume_detached(client, source_volume_name)
# Step-10
clone_volume = client.by_id_volume(clone_volume_name)
clone_volume.attach(hostId=lht_host_id)
wait_for_volume_attached(client, clone_volume_name)
clone_volume = wait_for_volume_endpoint(client, clone_volume_name)
# Step-11
check_volume_data(clone_volume, data)
# Step-12
wait_for_volume_healthy(client, clone_volume_name)
