def snapshot_create_and_record_md5sum(client, core_api, volume_name, pod_name, snapshots_md5sum): # NOQA data_md5sum = read_data_md5sum(core_api, pod_name) snap = create_snapshot(client, volume_name) snap_data = snapshot_data(snap["name"]) snap_data.set_data_md5sum(data_md5sum) snapshots_md5sum[snap["name"]] = snap_data return snap["name"]
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA vol_name = csi_pv['metadata']['name'] write_pod_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = create_snapshot(client, vol_name) volume.snapshotBackup(name=snap.name) common.wait_for_backup_completion(client, vol_name, snap.name) bv, b = common.find_backup(client, vol_name, snap.name) pod2_name = 'csi-backup-test-' + str(i) create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make, base_image, b.url) resp = read_volume_data(core_api, pod2_name) assert resp == test_data delete_backup(client, bv.name, b.name)
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, vol_name, backing_image, test_data): # NOQA write_pod_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = create_snapshot(client, vol_name) volume.snapshotBackup(name=snap.name) common.wait_for_backup_completion(client, vol_name, snap.name) bv, b = common.find_backup(client, vol_name, snap.name) pod2_name = 'csi-backup-test-2' vol2_name = create_and_wait_csi_pod( pod2_name, client, core_api, csi_pv, pvc, pod_make, backing_image, b.url) volume2 = client.by_id_volume(vol2_name) resp = read_volume_data(core_api, pod2_name) assert resp == test_data delete_backup(client, bv.name, b.name) delete_and_wait_pod(core_api, pod2_name) client.delete(volume2)
def test_backup_kubernetes_status(set_random_backupstore, client, core_api, pod): # NOQA """ Test that Backups have KubernetesStatus stored properly when there is an associated PersistentVolumeClaim and Pod. 1. Setup a random backupstore 2. Set settings Longhorn Static StorageClass to `longhorn-static-test` 3. Create a volume and PV/PVC. Verify the StorageClass of PVC 4. Create a Pod using the PVC. 5. Check volume's Kubernetes status to reflect PV/PVC/Pod correctly. 6. Create a backup for the volume. 7. Verify the labels of created backup reflect PV/PVC/Pod status. 8. Restore the backup to a volume. Wait for restoration to complete. 9. Check the volume's Kubernetes Status 1. Make sure the `lastPodRefAt` and `lastPVCRefAt` is snapshot created time 10. Delete the backup and restored volume. 11. Delete PV/PVC/Pod. 12. Verify volume's Kubernetes Status updated to reflect history data. 13. Attach the volume and create another backup. Verify the labels 14. Verify the volume's Kubernetes status. 15. Restore the previous backup to a new volume. Wait for restoration. 16. Verify the restored volume's Kubernetes status. 1. Make sure `lastPodRefAt` and `lastPVCRefAt` matched volume on step 12 """ host_id = get_self_host_id() static_sc_name = "longhorn-static-test" setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC) setting = client.update(setting, value=static_sc_name) assert setting.value == static_sc_name volume_name = "test-backup-kubernetes-status-pod" # NOQA client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = wait_for_volume_detached(client, volume_name) pod_name = "pod-" + volume_name pv_name = "pv-" + volume_name pvc_name = "pvc-" + volume_name create_pv_for_volume(client, core_api, volume, pv_name) create_pvc_for_volume(client, core_api, volume, pvc_name) ret = core_api.list_namespaced_persistent_volume_claim(namespace='default') pvc_found = False for item in ret.items: if item.metadata.name == pvc_name: pvc_found = item break assert pvc_found assert pvc_found.spec.storage_class_name == static_sc_name pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [{ 'name': pod['spec']['containers'][0]['volumeMounts'][0]['name'], 'persistentVolumeClaim': { 'claimName': pvc_name, }, }] create_and_wait_pod(core_api, pod) ks = { 'lastPodRefAt': '', 'lastPVCRefAt': '', 'namespace': 'default', 'pvcName': pvc_name, 'pvName': pv_name, 'pvStatus': 'Bound', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, volume_name, ks) volume = wait_for_volume_healthy(client, volume_name) # Create Backup manually instead of calling create_backup since Kubernetes # is not guaranteed to mount our Volume to the test host. snap = create_snapshot(client, volume_name) volume.snapshotBackup(name=snap.name) wait_for_backup_completion(client, volume_name, snap.name) _, b = find_backup(client, volume_name, snap.name) # Check backup label status = loads(b.labels.get(KUBERNETES_STATUS_LABEL)) assert status == ks # Check backup volume label for _ in range(RETRY_COUNTS): bv = client.by_id_backupVolume(volume_name) if bv is not None and bv.labels is not None: break time.sleep(RETRY_INTERVAL) assert bv is not None and bv.labels is not None status = loads(bv.labels.get(KUBERNETES_STATUS_LABEL)) assert status == ks restore_name = generate_volume_name() client.create_volume(name=restore_name, size=SIZE, numberOfReplicas=2, fromBackup=b.url) wait_for_volume_restoration_completed(client, restore_name) wait_for_volume_detached(client, restore_name) snapshot_created = b.snapshotCreated ks = { 'lastPodRefAt': b.snapshotCreated, 'lastPVCRefAt': b.snapshotCreated, 'namespace': 'default', 'pvcName': pvc_name, # Restoration should not apply PersistentVolume data. 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, restore_name, ks) restore = client.by_id_volume(restore_name) # We need to compare LastPodRefAt and LastPVCRefAt manually since # wait_volume_kubernetes_status only checks for empty or non-empty state. assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"] assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"] delete_backup(client, bv.name, b.name) client.delete(restore) wait_for_volume_delete(client, restore_name) delete_and_wait_pod(core_api, pod_name) delete_and_wait_pvc(core_api, pvc_name) delete_and_wait_pv(core_api, pv_name) # With the Pod, PVC, and PV deleted, the Volume should have both Ref # fields set. Check that a new Backup and Restore will use this instead of # manually populating the Ref fields. ks = { 'lastPodRefAt': 'NOT NULL', 'lastPVCRefAt': 'NOT NULL', 'namespace': 'default', 'pvcName': pvc_name, 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, volume_name, ks) volume = wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) snap = create_snapshot(client, volume_name) volume.snapshotBackup(name=snap.name) volume = wait_for_backup_completion(client, volume_name, snap.name) bv, b = find_backup(client, volume_name, snap.name) new_b = bv.backupGet(name=b.name) status = loads(new_b.labels.get(KUBERNETES_STATUS_LABEL)) # Check each field manually, we have no idea what the LastPodRefAt or the # LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated. assert status['lastPodRefAt'] != snapshot_created assert status['lastPVCRefAt'] != snapshot_created assert status['namespace'] == "default" assert status['pvcName'] == pvc_name assert status['pvName'] == "" assert status['pvStatus'] == "" assert status['workloadsStatus'] == [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] restore_name = generate_volume_name() client.create_volume(name=restore_name, size=SIZE, numberOfReplicas=2, fromBackup=b.url) wait_for_volume_restoration_completed(client, restore_name) wait_for_volume_detached(client, restore_name) ks = { 'lastPodRefAt': status['lastPodRefAt'], 'lastPVCRefAt': status['lastPVCRefAt'], 'namespace': 'default', 'pvcName': pvc_name, 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, restore_name, ks) restore = client.by_id_volume(restore_name) assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"] assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"] # cleanup backupstore_cleanup(client) client.delete(restore) cleanup_volume(client, volume)
def create_and_test_backups(api, cli, pod_info): """ Create backups for all Pods in a StatefulSet and tests that all the backups have the correct attributes. Args: api: An instance of CoreV1Api. cli: A Longhorn client instance. pod_info: A List of Pods with names and volume information. This List can be generated using the get_statefulset_pod_info function located in common.py. """ for pod in pod_info: pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE) pod['backup_snapshot'] = '' # Create backup. volume_name = pod['pv_name'] volume = cli.by_id_volume(pod['pv_name']) create_snapshot(cli, volume_name) write_pod_volume_data(api, pod['pod_name'], pod['data']) pod['backup_snapshot'] = create_snapshot(cli, volume_name) create_snapshot(cli, volume_name) volume.snapshotBackup(name=pod['backup_snapshot']['name']) # Wait for backup to appear. found = False for i in range(DEFAULT_BACKUP_TIMEOUT): backup_volumes = cli.list_backupVolume() for bv in backup_volumes: if bv.name == pod['pv_name']: found = True break if found: break time.sleep(DEFAULT_POD_INTERVAL) assert found found = False for i in range(DEFAULT_BACKUP_TIMEOUT): backups = bv.backupList().data for b in backups: if b['snapshotName'] == pod['backup_snapshot']['name']: found = True break if found: break time.sleep(DEFAULT_POD_INTERVAL) assert found # Make sure backup has the correct attributes. new_b = bv.backupGet(name=b.name) assert new_b.name == b.name assert new_b.url == b.url assert new_b.snapshotName == b.snapshotName assert new_b.snapshotCreated == b.snapshotCreated assert new_b.created == b.created assert new_b.volumeName == b.volumeName assert new_b.volumeSize == b.volumeSize assert new_b.volumeCreated == b.volumeCreated # This backup has the url attribute we need to restore from backup. pod['backup_snapshot'] = b
def test_restore_rwo_volume_to_rwx(set_random_backupstore, client, core_api, volume_name, pvc, csi_pv, pod_make, make_deployment_with_pvc): # NOQA """ Test restoring a rwo to a rwx volume. 1. Create a volume with 'accessMode' rwo. 2. Create a PV and a PVC with access mode 'readwriteonce' and attach to the volume. 3. Create a pod and attach to the PVC. 4. Write some data into the pod and compute md5sum. 5. Take a backup of the volume. 6. Restore the backup with 'accessMode' rwx. 7. Create PV and PVC and attach to 2 pods. 8. Verify the data. """ data_path = "/data/test" pod_name, pv_name, pvc_name, md5sum = \ prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc, pod_make, volume_name, data_size_in_mb=DATA_SIZE_IN_MB_1, data_path=data_path) snap = create_snapshot(client, volume_name) volume = client.by_id_volume(volume_name) volume.snapshotBackup(name=snap.name) wait_for_backup_completion(client, volume_name, snap.name) bv, b1 = find_backup(client, volume_name, snap.name) restore_volume_name = 'restored-rwx-volume' restore_pv_name = restore_volume_name + "-pv" restore_pvc_name = restore_volume_name + "-pvc" client.create_volume(name=restore_volume_name, size=str(1 * Gi), numberOfReplicas=3, fromBackup=b1.url, accessMode='rwx') wait_for_volume_creation(client, restore_volume_name) restore_volume = wait_for_volume_detached(client, restore_volume_name) create_pv_for_volume(client, core_api, restore_volume, restore_pv_name) create_pvc_for_volume(client, core_api, restore_volume, restore_pvc_name) deployment = make_deployment_with_pvc('deployment-multi-pods-test', restore_pvc_name, replicas=2) apps_api = get_apps_api_client() create_and_wait_deployment(apps_api, deployment) deployment_label_selector = \ "name=" + deployment["metadata"]["labels"]["name"] deployment_pod_list = \ core_api.list_namespaced_pod(namespace="default", label_selector=deployment_label_selector) pod_name_1 = deployment_pod_list.items[0].metadata.name pod_name_2 = deployment_pod_list.items[1].metadata.name md5sum_pod1 = get_pod_data_md5sum(core_api, pod_name_1, data_path) md5sum_pod2 = get_pod_data_md5sum(core_api, pod_name_2, data_path) assert md5sum == md5sum_pod1 == md5sum_pod2
def test_csi_volumesnapshot_restore_existing_backup( set_random_backupstore, # NOQA client, # NOQA core_api, # NOQA volume_name, # NOQA csi_pv, # NOQA pvc, # NOQA pod_make, # NOQA volumesnapshotclass, # NOQA volumesnapshotcontent, volumesnapshot, # NOQA volsnapshotclass_delete_policy, # NOQA backup_is_deleted): # NOQA """ Test retention of a backup while deleting the associated `VolumeSnapshot` via the csi snapshotter Context: We want to allow the user to programmatically create/restore/delete longhorn backups via the csi snapshot mechanism ref: https://kubernetes.io/docs/concepts/storage/volume-snapshots/ Setup: 1. Make sure your cluster contains the below crds https://github.com/kubernetes-csi/external-snapshotter /tree/master/client/config/crd 2. Make sure your cluster contains the snapshot controller https://github.com/kubernetes-csi/external-snapshotter /tree/master/deploy/kubernetes/snapshot-controller Steps: 1. create new snapshotClass with deletionPolicy set to Retain 2. call csi_volumesnapshot_creation_test(snapshotClass=custom) 3. call csi_volumesnapshot_restore_test() 4. call csi_volumesnapshot_deletion_test(deletionPolicy='Retain'): 5. cleanup """ csisnapclass = \ volumesnapshotclass(name="snapshotclass", deletepolicy=volsnapshotclass_delete_policy) pod_name, pv_name, pvc_name, md5sum = \ prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc, pod_make, volume_name, data_path="/data/test") volume = client.by_id_volume(volume_name) snap = create_snapshot(client, volume_name) volume.snapshotBackup(name=snap.name) wait_for_backup_completion(client, volume_name, snap.name) bv, b = find_backup(client, volume_name, snap.name) csivolsnap_name = volume_name + "-volumesnapshot" csivolsnap_namespace = "default" volsnapcontent = \ volumesnapshotcontent("volsnapcontent", csisnapclass["metadata"]["name"], "Delete", "bs://" + volume_name + "/" + b.name, csivolsnap_name, csivolsnap_namespace) csivolsnap = volumesnapshot(csivolsnap_name, csivolsnap_namespace, csisnapclass["metadata"]["name"], "volumeSnapshotContentName", volsnapcontent["metadata"]["name"]) restore_pvc_name = pvc["metadata"]["name"] + "-restore" restore_pvc_size = pvc["spec"]["resources"]["requests"]["storage"] restore_csi_volume_snapshot(core_api, client, csivolsnap, restore_pvc_name, restore_pvc_size) restore_pod = pod_make() restore_pod_name = restore_pod["metadata"]["name"] restore_pod['spec']['volumes'] = [create_pvc_spec(restore_pvc_name)] create_and_wait_pod(core_api, restore_pod) restore_md5sum = \ get_pod_data_md5sum(core_api, restore_pod_name, path="/data/test") assert restore_md5sum == md5sum # Delete volumeSnapshot test delete_volumesnapshot(csivolsnap["metadata"]["name"], "default") if backup_is_deleted is False: find_backup(client, volume_name, b["snapshotName"]) else: wait_for_backup_delete(client, volume_name, b["name"])