def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA vol_name = csi_pv['metadata']['name'] write_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = common.find_backup(client, vol_name, snap["name"]) pod2_name = 'csi-backup-test-' + str(i) create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make, base_image, b["url"]) resp = read_volume_data(core_api, pod2_name) assert resp == test_data bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap["name"]: found = True break assert not found
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA vol_name = csi_pv['metadata']['name'] write_pod_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = common.find_backup(client, vol_name, snap["name"]) pod2_name = 'csi-backup-test-' + str(i) create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make, base_image, b["url"]) resp = read_volume_data(core_api, pod2_name) assert resp == test_data bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap["name"]: found = True break assert not found
def create_backup(client, volname, data={}): volume = client.by_id_volume(volname) volume.snapshotCreate() if not data: data = write_volume_random_data(volume) else: data = write_volume_data(volume, data) snap = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = common.find_backup(client, volname, snap["name"]) new_b = bv.backupGet(name=b["name"]) assert new_b["name"] == b["name"] assert new_b["url"] == b["url"] assert new_b["snapshotName"] == b["snapshotName"] assert new_b["snapshotCreated"] == b["snapshotCreated"] assert new_b["created"] == b["created"] assert new_b["volumeName"] == b["volumeName"] assert new_b["volumeSize"] == b["volumeSize"] assert new_b["volumeCreated"] == b["volumeCreated"] volume = wait_for_volume_status(client, volname, "lastBackup", b["name"]) assert volume["lastBackupAt"] != "" return bv, b, snap, data
def backupstore_test(client, host_id, volname, size): volume = client.by_id_volume(volname) volume.snapshotCreate() data = write_volume_random_data(volume) snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) bv, b = common.find_backup(client, volname, snap2["name"]) new_b = bv.backupGet(name=b["name"]) assert new_b["name"] == b["name"] assert new_b["url"] == b["url"] assert new_b["snapshotName"] == b["snapshotName"] assert new_b["snapshotCreated"] == b["snapshotCreated"] assert new_b["created"] == b["created"] assert new_b["volumeName"] == b["volumeName"] assert new_b["volumeSize"] == b["volumeSize"] assert new_b["volumeCreated"] == b["volumeCreated"] # test restore restoreName = generate_volume_name() volume = client.create_volume(name=restoreName, size=size, numberOfReplicas=2, fromBackup=b["url"]) volume = common.wait_for_volume_detached(client, restoreName) assert volume["name"] == restoreName assert volume["size"] == size assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, restoreName) check_volume_data(volume, data) volume = volume.detach() volume = common.wait_for_volume_detached(client, restoreName) client.delete(volume) volume = wait_for_volume_delete(client, restoreName) bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap2["name"]: found = True break assert not found
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA vol_name = csi_pv['metadata']['name'] write_pod_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = create_snapshot(client, vol_name) volume.snapshotBackup(name=snap.name) common.wait_for_backup_completion(client, vol_name, snap.name) bv, b = common.find_backup(client, vol_name, snap.name) pod2_name = 'csi-backup-test-' + str(i) create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make, base_image, b.url) resp = read_volume_data(core_api, pod2_name) assert resp == test_data delete_backup(client, bv.name, b.name)
def backup_create_and_record_md5sum(client, core_api, volume_name, pod_name, snapshots_md5sum): # NOQA volume = client.by_id_volume(volume_name) data_md5sum = read_data_md5sum(core_api, pod_name) snap_name = snapshot_create_and_record_md5sum(client, core_api, volume_name, pod_name, snapshots_md5sum) snap = snapshot_data(snap_name) snapshots_md5sum[snap_name] = snap volume.snapshotBackup(name=snap_name) global WAIT_BACKUP_COMPLETE if WAIT_BACKUP_COMPLETE is None: WAIT_BACKUP_COMPLETE = bool(random.getrandbits(1)) if WAIT_BACKUP_COMPLETE is True: # wait for volume backupStatus to be updated with new backup record for i in range(RETRY_COUNTS): volume = client.by_id_volume(volume_name) for b in volume.backupStatus: if b.snapshot == snap_name: if b.error == "": wait_for_backup_completion(client, volume_name, snap_name) break else: print("...aborting backup " + b.error) return time.sleep(RETRY_INTERVAL) _, b = find_backup(client, volume_name, snap_name) snap.set_backup_name(b["name"]) snap.set_backup_url(b["url"]) snap.set_data_md5sum(data_md5sum)
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, vol_name, backing_image, test_data): # NOQA write_pod_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = create_snapshot(client, vol_name) volume.snapshotBackup(name=snap.name) common.wait_for_backup_completion(client, vol_name, snap.name) bv, b = common.find_backup(client, vol_name, snap.name) pod2_name = 'csi-backup-test-2' vol2_name = create_and_wait_csi_pod( pod2_name, client, core_api, csi_pv, pvc, pod_make, backing_image, b.url) volume2 = client.by_id_volume(vol2_name) resp = read_volume_data(core_api, pod2_name) assert resp == test_data delete_backup(client, bv.name, b.name) delete_and_wait_pod(core_api, pod2_name) client.delete(volume2)
def test_backup_kubernetes_status(set_random_backupstore, client, core_api, pod): # NOQA """ Test that Backups have KubernetesStatus stored properly when there is an associated PersistentVolumeClaim and Pod. 1. Setup a random backupstore 2. Set settings Longhorn Static StorageClass to `longhorn-static-test` 3. Create a volume and PV/PVC. Verify the StorageClass of PVC 4. Create a Pod using the PVC. 5. Check volume's Kubernetes status to reflect PV/PVC/Pod correctly. 6. Create a backup for the volume. 7. Verify the labels of created backup reflect PV/PVC/Pod status. 8. Restore the backup to a volume. Wait for restoration to complete. 9. Check the volume's Kubernetes Status 1. Make sure the `lastPodRefAt` and `lastPVCRefAt` is snapshot created time 10. Delete the backup and restored volume. 11. Delete PV/PVC/Pod. 12. Verify volume's Kubernetes Status updated to reflect history data. 13. Attach the volume and create another backup. Verify the labels 14. Verify the volume's Kubernetes status. 15. Restore the previous backup to a new volume. Wait for restoration. 16. Verify the restored volume's Kubernetes status. 1. Make sure `lastPodRefAt` and `lastPVCRefAt` matched volume on step 12 """ host_id = get_self_host_id() static_sc_name = "longhorn-static-test" setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC) setting = client.update(setting, value=static_sc_name) assert setting.value == static_sc_name volume_name = "test-backup-kubernetes-status-pod" # NOQA client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = wait_for_volume_detached(client, volume_name) pod_name = "pod-" + volume_name pv_name = "pv-" + volume_name pvc_name = "pvc-" + volume_name create_pv_for_volume(client, core_api, volume, pv_name) create_pvc_for_volume(client, core_api, volume, pvc_name) ret = core_api.list_namespaced_persistent_volume_claim(namespace='default') pvc_found = False for item in ret.items: if item.metadata.name == pvc_name: pvc_found = item break assert pvc_found assert pvc_found.spec.storage_class_name == static_sc_name pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [{ 'name': pod['spec']['containers'][0]['volumeMounts'][0]['name'], 'persistentVolumeClaim': { 'claimName': pvc_name, }, }] create_and_wait_pod(core_api, pod) ks = { 'lastPodRefAt': '', 'lastPVCRefAt': '', 'namespace': 'default', 'pvcName': pvc_name, 'pvName': pv_name, 'pvStatus': 'Bound', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, volume_name, ks) volume = wait_for_volume_healthy(client, volume_name) # Create Backup manually instead of calling create_backup since Kubernetes # is not guaranteed to mount our Volume to the test host. snap = create_snapshot(client, volume_name) volume.snapshotBackup(name=snap.name) wait_for_backup_completion(client, volume_name, snap.name) _, b = find_backup(client, volume_name, snap.name) # Check backup label status = loads(b.labels.get(KUBERNETES_STATUS_LABEL)) assert status == ks # Check backup volume label for _ in range(RETRY_COUNTS): bv = client.by_id_backupVolume(volume_name) if bv is not None and bv.labels is not None: break time.sleep(RETRY_INTERVAL) assert bv is not None and bv.labels is not None status = loads(bv.labels.get(KUBERNETES_STATUS_LABEL)) assert status == ks restore_name = generate_volume_name() client.create_volume(name=restore_name, size=SIZE, numberOfReplicas=2, fromBackup=b.url) wait_for_volume_restoration_completed(client, restore_name) wait_for_volume_detached(client, restore_name) snapshot_created = b.snapshotCreated ks = { 'lastPodRefAt': b.snapshotCreated, 'lastPVCRefAt': b.snapshotCreated, 'namespace': 'default', 'pvcName': pvc_name, # Restoration should not apply PersistentVolume data. 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, restore_name, ks) restore = client.by_id_volume(restore_name) # We need to compare LastPodRefAt and LastPVCRefAt manually since # wait_volume_kubernetes_status only checks for empty or non-empty state. assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"] assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"] delete_backup(client, bv.name, b.name) client.delete(restore) wait_for_volume_delete(client, restore_name) delete_and_wait_pod(core_api, pod_name) delete_and_wait_pvc(core_api, pvc_name) delete_and_wait_pv(core_api, pv_name) # With the Pod, PVC, and PV deleted, the Volume should have both Ref # fields set. Check that a new Backup and Restore will use this instead of # manually populating the Ref fields. ks = { 'lastPodRefAt': 'NOT NULL', 'lastPVCRefAt': 'NOT NULL', 'namespace': 'default', 'pvcName': pvc_name, 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, volume_name, ks) volume = wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) snap = create_snapshot(client, volume_name) volume.snapshotBackup(name=snap.name) volume = wait_for_backup_completion(client, volume_name, snap.name) bv, b = find_backup(client, volume_name, snap.name) new_b = bv.backupGet(name=b.name) status = loads(new_b.labels.get(KUBERNETES_STATUS_LABEL)) # Check each field manually, we have no idea what the LastPodRefAt or the # LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated. assert status['lastPodRefAt'] != snapshot_created assert status['lastPVCRefAt'] != snapshot_created assert status['namespace'] == "default" assert status['pvcName'] == pvc_name assert status['pvName'] == "" assert status['pvStatus'] == "" assert status['workloadsStatus'] == [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] restore_name = generate_volume_name() client.create_volume(name=restore_name, size=SIZE, numberOfReplicas=2, fromBackup=b.url) wait_for_volume_restoration_completed(client, restore_name) wait_for_volume_detached(client, restore_name) ks = { 'lastPodRefAt': status['lastPodRefAt'], 'lastPVCRefAt': status['lastPVCRefAt'], 'namespace': 'default', 'pvcName': pvc_name, 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, restore_name, ks) restore = client.by_id_volume(restore_name) assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"] assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"] # cleanup backupstore_cleanup(client) client.delete(restore) cleanup_volume(client, volume)
def test_restore_rwo_volume_to_rwx(set_random_backupstore, client, core_api, volume_name, pvc, csi_pv, pod_make, make_deployment_with_pvc): # NOQA """ Test restoring a rwo to a rwx volume. 1. Create a volume with 'accessMode' rwo. 2. Create a PV and a PVC with access mode 'readwriteonce' and attach to the volume. 3. Create a pod and attach to the PVC. 4. Write some data into the pod and compute md5sum. 5. Take a backup of the volume. 6. Restore the backup with 'accessMode' rwx. 7. Create PV and PVC and attach to 2 pods. 8. Verify the data. """ data_path = "/data/test" pod_name, pv_name, pvc_name, md5sum = \ prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc, pod_make, volume_name, data_size_in_mb=DATA_SIZE_IN_MB_1, data_path=data_path) snap = create_snapshot(client, volume_name) volume = client.by_id_volume(volume_name) volume.snapshotBackup(name=snap.name) wait_for_backup_completion(client, volume_name, snap.name) bv, b1 = find_backup(client, volume_name, snap.name) restore_volume_name = 'restored-rwx-volume' restore_pv_name = restore_volume_name + "-pv" restore_pvc_name = restore_volume_name + "-pvc" client.create_volume(name=restore_volume_name, size=str(1 * Gi), numberOfReplicas=3, fromBackup=b1.url, accessMode='rwx') wait_for_volume_creation(client, restore_volume_name) restore_volume = wait_for_volume_detached(client, restore_volume_name) create_pv_for_volume(client, core_api, restore_volume, restore_pv_name) create_pvc_for_volume(client, core_api, restore_volume, restore_pvc_name) deployment = make_deployment_with_pvc('deployment-multi-pods-test', restore_pvc_name, replicas=2) apps_api = get_apps_api_client() create_and_wait_deployment(apps_api, deployment) deployment_label_selector = \ "name=" + deployment["metadata"]["labels"]["name"] deployment_pod_list = \ core_api.list_namespaced_pod(namespace="default", label_selector=deployment_label_selector) pod_name_1 = deployment_pod_list.items[0].metadata.name pod_name_2 = deployment_pod_list.items[1].metadata.name md5sum_pod1 = get_pod_data_md5sum(core_api, pod_name_1, data_path) md5sum_pod2 = get_pod_data_md5sum(core_api, pod_name_2, data_path) assert md5sum == md5sum_pod1 == md5sum_pod2
def test_csi_volumesnapshot_restore_existing_backup( set_random_backupstore, # NOQA client, # NOQA core_api, # NOQA volume_name, # NOQA csi_pv, # NOQA pvc, # NOQA pod_make, # NOQA volumesnapshotclass, # NOQA volumesnapshotcontent, volumesnapshot, # NOQA volsnapshotclass_delete_policy, # NOQA backup_is_deleted): # NOQA """ Test retention of a backup while deleting the associated `VolumeSnapshot` via the csi snapshotter Context: We want to allow the user to programmatically create/restore/delete longhorn backups via the csi snapshot mechanism ref: https://kubernetes.io/docs/concepts/storage/volume-snapshots/ Setup: 1. Make sure your cluster contains the below crds https://github.com/kubernetes-csi/external-snapshotter /tree/master/client/config/crd 2. Make sure your cluster contains the snapshot controller https://github.com/kubernetes-csi/external-snapshotter /tree/master/deploy/kubernetes/snapshot-controller Steps: 1. create new snapshotClass with deletionPolicy set to Retain 2. call csi_volumesnapshot_creation_test(snapshotClass=custom) 3. call csi_volumesnapshot_restore_test() 4. call csi_volumesnapshot_deletion_test(deletionPolicy='Retain'): 5. cleanup """ csisnapclass = \ volumesnapshotclass(name="snapshotclass", deletepolicy=volsnapshotclass_delete_policy) pod_name, pv_name, pvc_name, md5sum = \ prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc, pod_make, volume_name, data_path="/data/test") volume = client.by_id_volume(volume_name) snap = create_snapshot(client, volume_name) volume.snapshotBackup(name=snap.name) wait_for_backup_completion(client, volume_name, snap.name) bv, b = find_backup(client, volume_name, snap.name) csivolsnap_name = volume_name + "-volumesnapshot" csivolsnap_namespace = "default" volsnapcontent = \ volumesnapshotcontent("volsnapcontent", csisnapclass["metadata"]["name"], "Delete", "bs://" + volume_name + "/" + b.name, csivolsnap_name, csivolsnap_namespace) csivolsnap = volumesnapshot(csivolsnap_name, csivolsnap_namespace, csisnapclass["metadata"]["name"], "volumeSnapshotContentName", volsnapcontent["metadata"]["name"]) restore_pvc_name = pvc["metadata"]["name"] + "-restore" restore_pvc_size = pvc["spec"]["resources"]["requests"]["storage"] restore_csi_volume_snapshot(core_api, client, csivolsnap, restore_pvc_name, restore_pvc_size) restore_pod = pod_make() restore_pod_name = restore_pod["metadata"]["name"] restore_pod['spec']['volumes'] = [create_pvc_spec(restore_pvc_name)] create_and_wait_pod(core_api, restore_pod) restore_md5sum = \ get_pod_data_md5sum(core_api, restore_pod_name, path="/data/test") assert restore_md5sum == md5sum # Delete volumeSnapshot test delete_volumesnapshot(csivolsnap["metadata"]["name"], "default") if backup_is_deleted is False: find_backup(client, volume_name, b["snapshotName"]) else: wait_for_backup_delete(client, volume_name, b["name"])
def test_csi_volumesnapshot_basic( set_random_backupstore, # NOQA volumesnapshotclass, # NOQA volumesnapshot, # NOQA client, # NOQA core_api, # NOQA volume_name, # NOQA csi_pv, # NOQA pvc, # NOQA pod_make, # NOQA volsnapshotclass_delete_policy, # NOQA backup_is_deleted): # NOQA """ Test creation / restoration / deletion of a backup via the csi snapshotter Context: We want to allow the user to programmatically create/restore/delete longhorn backups via the csi snapshot mechanism ref: https://kubernetes.io/docs/concepts/storage/volume-snapshots/ Setup: 1. Make sure your cluster contains the below crds https://github.com/kubernetes-csi/external-snapshotter /tree/master/client/config/crd 2. Make sure your cluster contains the snapshot controller https://github.com/kubernetes-csi/external-snapshotter /tree/master/deploy/kubernetes/snapshot-controller Steps: def csi_volumesnapshot_creation_test(snapshotClass=longhorn|custom): 1. create volume(1) 2. write data to volume(1) 3. create a kubernetes `VolumeSnapshot` object the `VolumeSnapshot.uuid` will be used to identify a **longhorn snapshot** and the associated `VolumeSnapshotContent` object 4. check creation of a new longhorn snapshot named `snapshot-uuid` 5. check for `VolumeSnapshotContent` named `snapcontent-uuid` 6. wait for `VolumeSnapshotContent.readyToUse` flag to be set to **true** 7. check for backup existance on the backupstore # the csi snapshot restore sets the fromBackup field same as # the StorageClass based restore approach. def csi_volumesnapshot_restore_test(): 8. create a `PersistentVolumeClaim` object where the `dataSource` field references the `VolumeSnapshot` object by name 9. verify creation of a new volume(2) bound to the pvc created in step(8) 10. verify data of new volume(2) equals data from backup (ie old data above) # default longhorn snapshot class is set to Delete # add a second test with a custom snapshot class with deletionPolicy # set to Retain you can reuse these methods for that and other tests def csi_volumesnapshot_deletion_test(deletionPolicy='Delete|Retain'): 11. delete `VolumeSnapshot` object 12. if deletionPolicy == Delete: 13. verify deletion of `VolumeSnapshot` and `VolumeSnapshotContent` objects 14. verify deletion of backup from backupstore 12. if deletionPolicy == Retain: 13. verify deletion of `VolumeSnapshot` 14. verify retention of `VolumeSnapshotContent` and backup on backupstore 15. cleanup """ csisnapclass = \ volumesnapshotclass(name="snapshotclass", deletepolicy=volsnapshotclass_delete_policy) pod_name, pv_name, pvc_name, md5sum = \ prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc, pod_make, volume_name, data_path="/data/test") # Create volumeSnapshot test csivolsnap = volumesnapshot(volume_name + "-volumesnapshot", "default", csisnapclass["metadata"]["name"], "persistentVolumeClaimName", pvc_name) volume = client.by_id_volume(volume_name) for i in range(RETRY_COUNTS): snapshots = volume.snapshotList() if len(snapshots) == 2: break time.sleep(RETRY_INTERVAL) lh_snapshot = None snapshots = volume.snapshotList() for snapshot in snapshots: if snapshot["name"] == "snapshot-" + csivolsnap["metadata"]["uid"]: lh_snapshot = snapshot assert lh_snapshot is not None wait_for_volumesnapshot_ready(csivolsnap["metadata"]["name"], csivolsnap["metadata"]["namespace"]) bv1, b = find_backup(client, volume_name, lh_snapshot["name"]) assert b["snapshotName"] == lh_snapshot["name"] restore_pvc_name = pvc["metadata"]["name"] + "-restore" restore_pvc_size = pvc["spec"]["resources"]["requests"]["storage"] restore_csi_volume_snapshot(core_api, client, csivolsnap, restore_pvc_name, restore_pvc_size) restore_pod = pod_make() restore_pod_name = restore_pod["metadata"]["name"] restore_pod['spec']['volumes'] = [create_pvc_spec(restore_pvc_name)] create_and_wait_pod(core_api, restore_pod) restore_md5sum = \ get_pod_data_md5sum(core_api, restore_pod_name, path="/data/test") assert restore_md5sum == md5sum # Delete volumeSnapshot test delete_volumesnapshot(csivolsnap["metadata"]["name"], "default") if backup_is_deleted is False: find_backup(client, volume_name, b["snapshotName"]) else: wait_for_backup_delete(client, volume_name, b["name"])
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" data = write_volume_random_data(volume) snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) _, b = common.find_backup(client, volume_name, snap2["name"]) res_name = common.generate_volume_name() res_volume = client.create_volume(name=res_name, size=size, numberOfReplicas=2, fromBackup=b["url"]) res_volume = common.wait_for_volume_restoration_completed( client, res_name) res_volume = common.wait_for_volume_detached(client, res_name) res_volume = res_volume.attach(hostId=host_id) res_volume = common.wait_for_volume_healthy(client, res_name) check_volume_data(res_volume, data) snapshots = res_volume.snapshotList() # only the backup snapshot + volume-head assert len(snapshots) == 2 backup_snapshot = "" for snap in snapshots: if snap["name"] != "volume-head": backup_snapshot = snap["name"] assert backup_snapshot != "" res_volume.snapshotCreate() snapshots = res_volume.snapshotList() assert len(snapshots) == 3 res_volume.snapshotDelete(name=backup_snapshot) res_volume.snapshotPurge() snapshots = res_volume.snapshotList() assert len(snapshots) == 2 ha_rebuild_replica_test(client, res_name) res_volume = res_volume.detach() res_volume = common.wait_for_volume_detached(client, res_name) client.delete(res_volume) common.wait_for_volume_delete(client, res_name) cleanup_volume(client, volume)
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" data = write_volume_random_data(volume) snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) _, b = common.find_backup(client, volume_name, snap2["name"]) res_name = common.generate_volume_name() res_volume = client.create_volume(name=res_name, size=size, numberOfReplicas=2, fromBackup=b["url"]) res_volume = common.wait_for_volume_detached(client, res_name) res_volume = res_volume.attach(hostId=host_id) res_volume = common.wait_for_volume_healthy(client, res_name) check_volume_data(res_volume, data) snapshots = res_volume.snapshotList() # only the backup snapshot + volume-head assert len(snapshots) == 2 backup_snapshot = "" for snap in snapshots: if snap["name"] != "volume-head": backup_snapshot = snap["name"] assert backup_snapshot != "" res_volume.snapshotCreate() snapshots = res_volume.snapshotList() assert len(snapshots) == 3 res_volume.snapshotDelete(name=backup_snapshot) res_volume.snapshotPurge() snapshots = res_volume.snapshotList() assert len(snapshots) == 2 ha_rebuild_replica_test(client, res_name) res_volume = res_volume.detach() res_volume = common.wait_for_volume_detached(client, res_name) client.delete(res_volume) common.wait_for_volume_delete(client, res_name) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_listing_backup_volume(clients, base_image=""): # NOQA for host_id, client in clients.iteritems(): break lht_hostId = get_self_host_id() # create 3 volumes. volume1_name = generate_volume_name() volume2_name = generate_volume_name() volume3_name = generate_volume_name() volume1 = create_and_check_volume(client, volume1_name) volume2 = create_and_check_volume(client, volume2_name) volume3 = create_and_check_volume(client, volume3_name) volume1.attach(hostId=lht_hostId) volume1 = common.wait_for_volume_healthy(client, volume1_name) volume2.attach(hostId=lht_hostId) volume2 = common.wait_for_volume_healthy(client, volume2_name) volume3.attach(hostId=lht_hostId) volume3 = common.wait_for_volume_healthy(client, volume3_name) # we only test NFS here. # Since it is difficult to directly remove volume.cfg from s3 buckets setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_nfs(backupstore): updated = False for i in range(RETRY_COMMAND_COUNT): nfs_url = backupstore.strip("nfs://") setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) if "nfs" in setting["value"]: updated = True break assert updated _, _, snap1, _ = create_backup(client, volume1_name) _, _, snap2, _ = create_backup(client, volume2_name) _, _, snap3, _ = create_backup(client, volume3_name) # invalidate backup volume 1 by renaming volume.cfg to volume.cfg.tmp cmd = ["mkdir", "-p", "/mnt/nfs"] subprocess.check_output(cmd) cmd = ["mount", "-t", "nfs4", nfs_url, "/mnt/nfs"] subprocess.check_output(cmd) cmd = ["find", "/mnt/nfs", "-type", "d", "-name", volume1_name] volume1_backup_volume_path = subprocess.check_output(cmd).strip() cmd = ["find", volume1_backup_volume_path, "-name", "volume.cfg"] volume1_backup_volume_cfg_path = subprocess.check_output(cmd).strip() cmd = [ "mv", volume1_backup_volume_cfg_path, volume1_backup_volume_cfg_path + ".tmp" ] subprocess.check_output(cmd) subprocess.check_output(["sync"]) found1 = found2 = found3 = False for i in range(RETRY_COUNTS): bvs = client.list_backupVolume() for bv in bvs: if bv["name"] == volume1_name: if "error" in bv.messages: assert "volume.cfg" in bv.messages["error"].lower() found1 = True elif bv["name"] == volume2_name: assert not bv.messages found2 = True elif bv["name"] == volume3_name: assert not bv.messages found3 = True if found1 & found2 & found3: break time.sleep(RETRY_INTERVAL) assert found1 & found2 & found3 cmd = [ "mv", volume1_backup_volume_cfg_path + ".tmp", volume1_backup_volume_cfg_path ] subprocess.check_output(cmd) subprocess.check_output(["sync"]) found = False for i in range(RETRY_COMMAND_COUNT): try: bv1, b1 = common.find_backup(client, volume1_name, snap1["name"]) found = True break except Exception: time.sleep(1) assert found bv1.backupDelete(name=b1["name"]) for i in range(RETRY_COMMAND_COUNT): found = False backups1 = bv1.backupList() for b in backups1: if b["snapshotName"] == snap1["name"]: found = True break assert not found bv2, b2 = common.find_backup(client, volume2_name, snap2["name"]) bv2.backupDelete(name=b2["name"]) for i in range(RETRY_COMMAND_COUNT): found = False backups2 = bv2.backupList() for b in backups2: if b["snapshotName"] == snap2["name"]: found = True break assert not found bv3, b3 = common.find_backup(client, volume3_name, snap3["name"]) bv3.backupDelete(name=b3["name"]) for i in range(RETRY_COMMAND_COUNT): found = False backups3 = bv3.backupList() for b in backups3: if b["snapshotName"] == snap3["name"]: found = True break assert not found volume1.detach() volume1 = common.wait_for_volume_detached(client, volume1_name) client.delete(volume1) wait_for_volume_delete(client, volume1_name) volume2.detach() volume2 = common.wait_for_volume_detached(client, volume2_name) client.delete(volume2) wait_for_volume_delete(client, volume2_name) volume3.detach() volume3 = common.wait_for_volume_detached(client, volume3_name) client.delete(volume3) wait_for_volume_delete(client, volume3_name) volumes = client.list_volume() assert len(volumes) == 0
def test_backup_kubernetes_status(client, core_api, pod): # NOQA """ Test that Backups have KubernetesStatus stored properly when there is an associated PersistentVolumeClaim and Pod. """ host_id = get_self_host_id() static_sc_name = "longhorn-static-test" setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC) setting = client.update(setting, value=static_sc_name) assert setting["value"] == static_sc_name volume_name = "test-backup-kubernetes-status-pod" client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = wait_for_volume_detached(client, volume_name) pod_name = "pod-" + volume_name pv_name = "pv-" + volume_name pvc_name = "pvc-" + volume_name create_pv_for_volume(client, core_api, volume, pv_name) create_pvc_for_volume(client, core_api, volume, pvc_name) ret = core_api.list_namespaced_persistent_volume_claim(namespace='default') pvc_found = False for item in ret.items: if item.metadata.name == pvc_name: pvc_found = item break assert pvc_found assert pvc_found.spec.storage_class_name == static_sc_name pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [{ 'name': pod['spec']['containers'][0]['volumeMounts'][0]['name'], 'persistentVolumeClaim': { 'claimName': pvc_name, }, }] create_and_wait_pod(core_api, pod) ks = { 'lastPodRefAt': '', 'lastPVCRefAt': '', 'namespace': 'default', 'pvcName': pvc_name, 'pvName': pv_name, 'pvStatus': 'Bound', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, volume_name, ks) volume = wait_for_volume_healthy(client, volume_name) # Create Backup manually instead of calling create_backup since Kubernetes # is not guaranteed to mount our Volume to the test host. snap = volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = find_backup(client, volume_name, snap["name"]) new_b = bv.backupGet(name=b["name"]) status = loads(new_b["labels"].get(KUBERNETES_STATUS_LABEL)) assert status == ks restore_name = generate_volume_name() client.create_volume(name=restore_name, size=SIZE, numberOfReplicas=2, fromBackup=b["url"]) wait_for_volume_restoration_completed(client, restore_name) wait_for_volume_detached(client, restore_name) snapshot_created = b["snapshotCreated"] ks = { 'lastPodRefAt': b["snapshotCreated"], 'lastPVCRefAt': b["snapshotCreated"], 'namespace': 'default', 'pvcName': pvc_name, # Restoration should not apply PersistentVolume data. 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, restore_name, ks) restore = client.by_id_volume(restore_name) # We need to compare LastPodRefAt and LastPVCRefAt manually since # wait_volume_kubernetes_status only checks for empty or non-empty state. assert restore["kubernetesStatus"]["lastPodRefAt"] == ks["lastPodRefAt"] assert restore["kubernetesStatus"]["lastPVCRefAt"] == ks["lastPVCRefAt"] bv.backupDelete(name=b["name"]) client.delete(restore) wait_for_volume_delete(client, restore_name) delete_and_wait_pod(core_api, pod_name) delete_and_wait_pvc(core_api, pvc_name) delete_and_wait_pv(core_api, pv_name) # With the Pod, PVC, and PV deleted, the Volume should have both Ref # fields set. Check that a new Backup and Restore will use this instead of # manually populating the Ref fields. ks = { 'lastPodRefAt': 'NOT NULL', 'lastPVCRefAt': 'NOT NULL', 'namespace': 'default', 'pvcName': pvc_name, 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, volume_name, ks) volume = wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) snap = volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = find_backup(client, volume_name, snap["name"]) new_b = bv.backupGet(name=b["name"]) status = loads(new_b["labels"].get(KUBERNETES_STATUS_LABEL)) # Check each field manually, we have no idea what the LastPodRefAt or the # LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated. assert status["lastPodRefAt"] != snapshot_created assert status["lastPVCRefAt"] != snapshot_created assert status["namespace"] == "default" assert status["pvcName"] == pvc_name assert status["pvName"] == "" assert status["pvStatus"] == "" assert status["workloadsStatus"] == [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] restore_name = generate_volume_name() client.create_volume(name=restore_name, size=SIZE, numberOfReplicas=2, fromBackup=b["url"]) wait_for_volume_restoration_completed(client, restore_name) wait_for_volume_detached(client, restore_name) ks = { 'lastPodRefAt': status["lastPodRefAt"], 'lastPVCRefAt': status["lastPVCRefAt"], 'namespace': 'default', 'pvcName': pvc_name, 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, restore_name, ks) restore = client.by_id_volume(restore_name) assert restore["kubernetesStatus"]["lastPodRefAt"] == ks["lastPodRefAt"] assert restore["kubernetesStatus"]["lastPVCRefAt"] == ks["lastPVCRefAt"] bv.backupDelete(name=b["name"]) client.delete(restore) cleanup_volume(client, volume)