def backupstore_cleanup(client):
backup_volumes = client.list_backup_volume()
for backup_volume in backup_volumes:
backups = backup_volume.backupList()
for backup in backups:
delete_backup(client, backup_volume.name, backup.name)
delete_backup_volume(client, backup_volume.name)
backup_volumes = client.list_backup_volume()
assert backup_volumes.data == []
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA
vol_name = csi_pv['metadata']['name']
write_pod_volume_data(core_api, pod_name, test_data)
volume = client.by_id_volume(vol_name)
snap = create_snapshot(client, vol_name)
volume.snapshotBackup(name=snap.name)
common.wait_for_backup_completion(client, vol_name, snap.name)
bv, b = common.find_backup(client, vol_name, snap.name)
pod2_name = 'csi-backup-test-' + str(i)
create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make,
base_image, b.url)
resp = read_volume_data(core_api, pod2_name)
assert resp == test_data
delete_backup(client, bv.name, b.name)
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, vol_name, backing_image, test_data): # NOQA
write_pod_volume_data(core_api, pod_name, test_data)
volume = client.by_id_volume(vol_name)
snap = create_snapshot(client, vol_name)
volume.snapshotBackup(name=snap.name)
common.wait_for_backup_completion(client, vol_name, snap.name)
bv, b = common.find_backup(client, vol_name, snap.name)
pod2_name = 'csi-backup-test-2'
vol2_name = create_and_wait_csi_pod(
pod2_name, client, core_api, csi_pv, pvc, pod_make,
backing_image, b.url)
volume2 = client.by_id_volume(vol2_name)
resp = read_volume_data(core_api, pod2_name)
assert resp == test_data
delete_backup(client, bv.name, b.name)
delete_and_wait_pod(core_api, pod2_name)
client.delete(volume2)
def test_backup_kubernetes_status(set_random_backupstore, client, core_api,
pod): # NOQA
"""
Test that Backups have KubernetesStatus stored properly when there is an
associated PersistentVolumeClaim and Pod.
1. Setup a random backupstore
2. Set settings Longhorn Static StorageClass to `longhorn-static-test`
3. Create a volume and PV/PVC. Verify the StorageClass of PVC
4. Create a Pod using the PVC.
5. Check volume's Kubernetes status to reflect PV/PVC/Pod correctly.
6. Create a backup for the volume.
7. Verify the labels of created backup reflect PV/PVC/Pod status.
8. Restore the backup to a volume. Wait for restoration to complete.
9. Check the volume's Kubernetes Status
1. Make sure the `lastPodRefAt` and `lastPVCRefAt` is snapshot created
time
10. Delete the backup and restored volume.
11. Delete PV/PVC/Pod.
12. Verify volume's Kubernetes Status updated to reflect history data.
13. Attach the volume and create another backup. Verify the labels
14. Verify the volume's Kubernetes status.
15. Restore the previous backup to a new volume. Wait for restoration.
16. Verify the restored volume's Kubernetes status.
1. Make sure `lastPodRefAt` and `lastPVCRefAt` matched volume on step
12
"""
host_id = get_self_host_id()
static_sc_name = "longhorn-static-test"
setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC)
setting = client.update(setting, value=static_sc_name)
assert setting.value == static_sc_name
volume_name = "test-backup-kubernetes-status-pod" # NOQA
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2)
volume = wait_for_volume_detached(client, volume_name)
pod_name = "pod-" + volume_name
pv_name = "pv-" + volume_name
pvc_name = "pvc-" + volume_name
create_pv_for_volume(client, core_api, volume, pv_name)
create_pvc_for_volume(client, core_api, volume, pvc_name)
ret = core_api.list_namespaced_persistent_volume_claim(namespace='default')
pvc_found = False
for item in ret.items:
if item.metadata.name == pvc_name:
pvc_found = item
break
assert pvc_found
assert pvc_found.spec.storage_class_name == static_sc_name
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pvc_name,
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'lastPodRefAt':
'',
'lastPVCRefAt':
'',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
pv_name,
'pvStatus':
'Bound',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_healthy(client, volume_name)
# Create Backup manually instead of calling create_backup since Kubernetes
# is not guaranteed to mount our Volume to the test host.
snap = create_snapshot(client, volume_name)
volume.snapshotBackup(name=snap.name)
wait_for_backup_completion(client, volume_name, snap.name)
_, b = find_backup(client, volume_name, snap.name)
# Check backup label
status = loads(b.labels.get(KUBERNETES_STATUS_LABEL))
assert status == ks
# Check backup volume label
for _ in range(RETRY_COUNTS):
bv = client.by_id_backupVolume(volume_name)
if bv is not None and bv.labels is not None:
break
time.sleep(RETRY_INTERVAL)
assert bv is not None and bv.labels is not None
status = loads(bv.labels.get(KUBERNETES_STATUS_LABEL))
assert status == ks
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b.url)
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
snapshot_created = b.snapshotCreated
ks = {
'lastPodRefAt':
b.snapshotCreated,
'lastPVCRefAt':
b.snapshotCreated,
'namespace':
'default',
'pvcName':
pvc_name,
# Restoration should not apply PersistentVolume data.
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
# We need to compare LastPodRefAt and LastPVCRefAt manually since
# wait_volume_kubernetes_status only checks for empty or non-empty state.
assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"]
assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"]
delete_backup(client, bv.name, b.name)
client.delete(restore)
wait_for_volume_delete(client, restore_name)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, pvc_name)
delete_and_wait_pv(core_api, pv_name)
# With the Pod, PVC, and PV deleted, the Volume should have both Ref
# fields set. Check that a new Backup and Restore will use this instead of
# manually populating the Ref fields.
ks = {
'lastPodRefAt':
'NOT NULL',
'lastPVCRefAt':
'NOT NULL',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
snap = create_snapshot(client, volume_name)
volume.snapshotBackup(name=snap.name)
volume = wait_for_backup_completion(client, volume_name, snap.name)
bv, b = find_backup(client, volume_name, snap.name)
new_b = bv.backupGet(name=b.name)
status = loads(new_b.labels.get(KUBERNETES_STATUS_LABEL))
# Check each field manually, we have no idea what the LastPodRefAt or the
# LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated.
assert status['lastPodRefAt'] != snapshot_created
assert status['lastPVCRefAt'] != snapshot_created
assert status['namespace'] == "default"
assert status['pvcName'] == pvc_name
assert status['pvName'] == ""
assert status['pvStatus'] == ""
assert status['workloadsStatus'] == [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b.url)
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
ks = {
'lastPodRefAt':
status['lastPodRefAt'],
'lastPVCRefAt':
status['lastPVCRefAt'],
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"]
assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"]
# cleanup
backupstore_cleanup(client)
client.delete(restore)
cleanup_volume(client, volume)