def test_backup_kubernetes_status(set_random_backupstore, client, core_api,
pod): # NOQA
"""
Test that Backups have KubernetesStatus stored properly when there is an
associated PersistentVolumeClaim and Pod.
1. Setup a random backupstore
2. Set settings Longhorn Static StorageClass to `longhorn-static-test`
3. Create a volume and PV/PVC. Verify the StorageClass of PVC
4. Create a Pod using the PVC.
5. Check volume's Kubernetes status to reflect PV/PVC/Pod correctly.
6. Create a backup for the volume.
7. Verify the labels of created backup reflect PV/PVC/Pod status.
8. Restore the backup to a volume. Wait for restoration to complete.
9. Check the volume's Kubernetes Status
1. Make sure the `lastPodRefAt` and `lastPVCRefAt` is snapshot created
time
10. Delete the backup and restored volume.
11. Delete PV/PVC/Pod.
12. Verify volume's Kubernetes Status updated to reflect history data.
13. Attach the volume and create another backup. Verify the labels
14. Verify the volume's Kubernetes status.
15. Restore the previous backup to a new volume. Wait for restoration.
16. Verify the restored volume's Kubernetes status.
1. Make sure `lastPodRefAt` and `lastPVCRefAt` matched volume on step
12
"""
host_id = get_self_host_id()
static_sc_name = "longhorn-static-test"
setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC)
setting = client.update(setting, value=static_sc_name)
assert setting.value == static_sc_name
volume_name = "test-backup-kubernetes-status-pod" # NOQA
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2)
volume = wait_for_volume_detached(client, volume_name)
pod_name = "pod-" + volume_name
pv_name = "pv-" + volume_name
pvc_name = "pvc-" + volume_name
create_pv_for_volume(client, core_api, volume, pv_name)
create_pvc_for_volume(client, core_api, volume, pvc_name)
ret = core_api.list_namespaced_persistent_volume_claim(namespace='default')
pvc_found = False
for item in ret.items:
if item.metadata.name == pvc_name:
pvc_found = item
break
assert pvc_found
assert pvc_found.spec.storage_class_name == static_sc_name
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pvc_name,
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'lastPodRefAt':
'',
'lastPVCRefAt':
'',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
pv_name,
'pvStatus':
'Bound',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_healthy(client, volume_name)
# Create Backup manually instead of calling create_backup since Kubernetes
# is not guaranteed to mount our Volume to the test host.
snap = create_snapshot(client, volume_name)
volume.snapshotBackup(name=snap.name)
wait_for_backup_completion(client, volume_name, snap.name)
_, b = find_backup(client, volume_name, snap.name)
# Check backup label
status = loads(b.labels.get(KUBERNETES_STATUS_LABEL))
assert status == ks
# Check backup volume label
for _ in range(RETRY_COUNTS):
bv = client.by_id_backupVolume(volume_name)
if bv is not None and bv.labels is not None:
break
time.sleep(RETRY_INTERVAL)
assert bv is not None and bv.labels is not None
status = loads(bv.labels.get(KUBERNETES_STATUS_LABEL))
assert status == ks
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b.url)
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
snapshot_created = b.snapshotCreated
ks = {
'lastPodRefAt':
b.snapshotCreated,
'lastPVCRefAt':
b.snapshotCreated,
'namespace':
'default',
'pvcName':
pvc_name,
# Restoration should not apply PersistentVolume data.
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
# We need to compare LastPodRefAt and LastPVCRefAt manually since
# wait_volume_kubernetes_status only checks for empty or non-empty state.
assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"]
assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"]
delete_backup(client, bv.name, b.name)
client.delete(restore)
wait_for_volume_delete(client, restore_name)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, pvc_name)
delete_and_wait_pv(core_api, pv_name)
# With the Pod, PVC, and PV deleted, the Volume should have both Ref
# fields set. Check that a new Backup and Restore will use this instead of
# manually populating the Ref fields.
ks = {
'lastPodRefAt':
'NOT NULL',
'lastPVCRefAt':
'NOT NULL',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
snap = create_snapshot(client, volume_name)
volume.snapshotBackup(name=snap.name)
volume = wait_for_backup_completion(client, volume_name, snap.name)
bv, b = find_backup(client, volume_name, snap.name)
new_b = bv.backupGet(name=b.name)
status = loads(new_b.labels.get(KUBERNETES_STATUS_LABEL))
# Check each field manually, we have no idea what the LastPodRefAt or the
# LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated.
assert status['lastPodRefAt'] != snapshot_created
assert status['lastPVCRefAt'] != snapshot_created
assert status['namespace'] == "default"
assert status['pvcName'] == pvc_name
assert status['pvName'] == ""
assert status['pvStatus'] == ""
assert status['workloadsStatus'] == [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b.url)
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
ks = {
'lastPodRefAt':
status['lastPodRefAt'],
'lastPVCRefAt':
status['lastPVCRefAt'],
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"]
assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"]
# cleanup
backupstore_cleanup(client)
client.delete(restore)
cleanup_volume(client, volume)
def test_pvc_creation_with_default_sc_set(client, core_api, storage_class,
pod): # NOQA
# set default storage class
storage_class['metadata']['annotations'] = \
{"storageclass.kubernetes.io/is-default-class": "true"}
create_storage_class(storage_class)
static_sc_name = "longhorn-static-test"
setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC)
setting = client.update(setting, value=static_sc_name)
assert setting["value"] == static_sc_name
volume_name = "test-pvc-creation-with-sc"
pod_name = "pod-" + volume_name
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2)
volume = wait_for_volume_detached(client, volume_name)
pv_name = "pv-" + volume_name
pvc_name = "pvc-" + volume_name
pvc_name_extra = "pvc-" + volume_name + "-extra"
create_pv_for_volume(client, core_api, volume, pv_name)
create_pvc_for_volume(client, core_api, volume, pvc_name)
ret = core_api.list_namespaced_persistent_volume_claim(namespace='default')
for item in ret.items:
if item.metadata.name == pvc_name:
pvc_found = item
break
assert pvc_found
assert pvc_found.spec.storage_class_name == static_sc_name
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pvc_name,
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'pvName':
pv_name,
'pvStatus':
'Bound',
'namespace':
'default',
'pvcName':
pvc_name,
'lastPVCRefAt':
'',
'lastPodRefAt':
'',
'workloadsStatus': [
{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
},
],
}
wait_volume_kubernetes_status(client, volume_name, ks)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, pvc_name)
# try to reuse the pv
volume = wait_for_volume_detached(client, volume_name)
create_pvc_for_volume(client, core_api, volume, pvc_name_extra)
pod['spec']['volumes'][0]['persistentVolumeClaim']['claimName'] = \
pvc_name_extra
create_and_wait_pod(core_api, pod)
ks['pvcName'] = pvc_name_extra
wait_volume_kubernetes_status(client, volume_name, ks)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, pvc_name_extra)
delete_and_wait_pv(core_api, pv_name)
# without default storage class
delete_storage_class(storage_class['metadata']['name'])
create_pv_for_volume(client, core_api, volume, pv_name)
create_pvc_for_volume(client, core_api, volume, pvc_name)
ret = core_api.list_namespaced_persistent_volume_claim(namespace='default')
for item in ret.items:
if item.metadata.name == pvc_name:
pvc2 = item
break
assert pvc2
assert pvc2.spec.storage_class_name == static_sc_name
delete_and_wait_pvc(core_api, pvc_name)
delete_and_wait_pv(core_api, pv_name)
def test_pvc_creation_with_default_sc_set(client, core_api, storage_class,
pod): # NOQA
"""
Test creating PVC with default StorageClass set
The target is to make sure the newly create PV/PVC won't use default
StorageClass, and if there is no default StorageClass, PV/PVC can still be
created.
1. Create a StorageClass and set it to be the default StorageClass
2. Update static StorageClass to `longhorn-static-test`
3. Create volume then PV/PVC.
4. Make sure the newly created PV/PVC using StorageClass
`longhorn-static-test`
5. Create pod with PVC.
6. Verify volume's Kubernetes Status
7. Remove PVC and Pod.
8. Verify volume's Kubernetes Status only contains current PV and history
9. Wait for volume to detach (since pod is deleted)
10. Reuse the volume on a new pod. Wait for the pod to start
11. Verify volume's Kubernetes Status reflect the new pod.
12. Delete PV/PVC/Pod.
13. Verify volume's Kubernetes Status only contains history
14. Delete the default StorageClass.
15. Create PV/PVC for the volume.
16. Make sure the PV's StorageClass is static StorageClass
"""
# set default storage class
storage_class['metadata']['annotations'] = \
{"storageclass.kubernetes.io/is-default-class": "true"}
create_storage_class(storage_class)
static_sc_name = "longhorn-static-test"
setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC)
setting = client.update(setting, value=static_sc_name)
assert setting.value == static_sc_name
volume_name = "test-pvc-creation-with-sc" # NOQA
pod_name = "pod-" + volume_name
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2)
volume = wait_for_volume_detached(client, volume_name)
pv_name = "pv-" + volume_name
pvc_name = "pvc-" + volume_name
pvc_name_extra = "pvc-" + volume_name + "-extra"
create_pv_for_volume(client, core_api, volume, pv_name)
create_pvc_for_volume(client, core_api, volume, pvc_name)
ret = core_api.list_namespaced_persistent_volume_claim(namespace='default')
for item in ret.items:
if item.metadata.name == pvc_name:
pvc_found = item
break
assert pvc_found
assert pvc_found.spec.storage_class_name == static_sc_name
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pvc_name,
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'pvName':
pv_name,
'pvStatus':
'Bound',
'namespace':
'default',
'pvcName':
pvc_name,
'lastPVCRefAt':
'',
'lastPodRefAt':
'',
'workloadsStatus': [
{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
},
],
}
wait_volume_kubernetes_status(client, volume_name, ks)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, pvc_name)
ks = {
'pvName': pv_name,
'pvStatus': 'Released',
'namespace': 'default',
'pvcName': pvc_name,
'lastPVCRefAt': 'not empty',
'lastPodRefAt': 'not empty',
}
wait_volume_kubernetes_status(client, volume_name, ks)
# try to reuse the pv
volume = wait_for_volume_detached(client, volume_name)
create_pvc_for_volume(client, core_api, volume, pvc_name_extra)
pod['spec']['volumes'][0]['persistentVolumeClaim']['claimName'] = \
pvc_name_extra
create_and_wait_pod(core_api, pod)
ks = {
'pvName':
pv_name,
'pvStatus':
'Bound',
'namespace':
'default',
'pvcName':
pvc_name_extra,
'lastPVCRefAt':
'',
'lastPodRefAt':
'',
'workloadsStatus': [
{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': '',
},
],
}
wait_volume_kubernetes_status(client, volume_name, ks)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, pvc_name_extra)
delete_and_wait_pv(core_api, pv_name)
ks = {
'pvName': '',
'pvStatus': '',
'namespace': 'default',
'pvcName': pvc_name_extra,
'lastPVCRefAt': 'not empty',
'lastPodRefAt': 'not empty',
}
wait_volume_kubernetes_status(client, volume_name, ks)
# without default storage class
delete_storage_class(storage_class['metadata']['name'])
create_pv_for_volume(client, core_api, volume, pv_name)
create_pvc_for_volume(client, core_api, volume, pvc_name)
ret = core_api.list_namespaced_persistent_volume_claim(namespace='default')
for item in ret.items:
if item.metadata.name == pvc_name:
pvc2 = item
break
assert pvc2
assert pvc2.spec.storage_class_name == static_sc_name
delete_and_wait_pvc(core_api, pvc_name)
delete_and_wait_pv(core_api, pv_name)
def test_backup_kubernetes_status(client, core_api, pod): # NOQA
"""
Test that Backups have KubernetesStatus stored properly when there is an
associated PersistentVolumeClaim and Pod.
"""
host_id = get_self_host_id()
static_sc_name = "longhorn-static-test"
setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC)
setting = client.update(setting, value=static_sc_name)
assert setting["value"] == static_sc_name
volume_name = "test-backup-kubernetes-status-pod"
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2)
volume = wait_for_volume_detached(client, volume_name)
pod_name = "pod-" + volume_name
pv_name = "pv-" + volume_name
pvc_name = "pvc-" + volume_name
create_pv_for_volume(client, core_api, volume, pv_name)
create_pvc_for_volume(client, core_api, volume, pvc_name)
ret = core_api.list_namespaced_persistent_volume_claim(namespace='default')
pvc_found = False
for item in ret.items:
if item.metadata.name == pvc_name:
pvc_found = item
break
assert pvc_found
assert pvc_found.spec.storage_class_name == static_sc_name
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pvc_name,
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'lastPodRefAt':
'',
'lastPVCRefAt':
'',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
pv_name,
'pvStatus':
'Bound',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_healthy(client, volume_name)
# Create Backup manually instead of calling create_backup since Kubernetes
# is not guaranteed to mount our Volume to the test host.
snap = volume.snapshotCreate()
volume.snapshotBackup(name=snap["name"])
bv, b = find_backup(client, volume_name, snap["name"])
new_b = bv.backupGet(name=b["name"])
status = loads(new_b["labels"].get(KUBERNETES_STATUS_LABEL))
assert status == ks
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b["url"])
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
snapshot_created = b["snapshotCreated"]
ks = {
'lastPodRefAt':
b["snapshotCreated"],
'lastPVCRefAt':
b["snapshotCreated"],
'namespace':
'default',
'pvcName':
pvc_name,
# Restoration should not apply PersistentVolume data.
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
# We need to compare LastPodRefAt and LastPVCRefAt manually since
# wait_volume_kubernetes_status only checks for empty or non-empty state.
assert restore["kubernetesStatus"]["lastPodRefAt"] == ks["lastPodRefAt"]
assert restore["kubernetesStatus"]["lastPVCRefAt"] == ks["lastPVCRefAt"]
bv.backupDelete(name=b["name"])
client.delete(restore)
wait_for_volume_delete(client, restore_name)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, pvc_name)
delete_and_wait_pv(core_api, pv_name)
# With the Pod, PVC, and PV deleted, the Volume should have both Ref
# fields set. Check that a new Backup and Restore will use this instead of
# manually populating the Ref fields.
ks = {
'lastPodRefAt':
'NOT NULL',
'lastPVCRefAt':
'NOT NULL',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
snap = volume.snapshotCreate()
volume.snapshotBackup(name=snap["name"])
bv, b = find_backup(client, volume_name, snap["name"])
new_b = bv.backupGet(name=b["name"])
status = loads(new_b["labels"].get(KUBERNETES_STATUS_LABEL))
# Check each field manually, we have no idea what the LastPodRefAt or the
# LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated.
assert status["lastPodRefAt"] != snapshot_created
assert status["lastPVCRefAt"] != snapshot_created
assert status["namespace"] == "default"
assert status["pvcName"] == pvc_name
assert status["pvName"] == ""
assert status["pvStatus"] == ""
assert status["workloadsStatus"] == [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b["url"])
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
ks = {
'lastPodRefAt':
status["lastPodRefAt"],
'lastPVCRefAt':
status["lastPVCRefAt"],
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
assert restore["kubernetesStatus"]["lastPodRefAt"] == ks["lastPodRefAt"]
assert restore["kubernetesStatus"]["lastPVCRefAt"] == ks["lastPVCRefAt"]
bv.backupDelete(name=b["name"])
client.delete(restore)
cleanup_volume(client, volume)
