def test_provisioner_io(client, core_api, storage_class, pvc, pod): # NOQA """ Test that input and output on a StorageClass provisioned PersistentVolumeClaim works as expected. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ # Prepare pod and volume specs. pod_name = 'provisioner-io-test' pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])] pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_storage(core_api, storage_class, pvc) create_and_wait_pod(core_api, pod) pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name']) write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) common.wait_for_volume_detached(client, pvc_volume_name) pod_name = 'flexvolume-provisioner-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def test_statefulset_pod_deletion(core_api, storage_class, statefulset): # NOQA """ Test that a StatefulSet can spin up a new Pod with the same data after a previous Pod has been deleted. This test will only work in a CSI environment. It will automatically be disabled in FlexVolume environments. """ statefulset_name = 'statefulset-pod-deletion-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) test_pod_name = statefulset_name + '-' + \ str(randrange(statefulset['spec']['replicas'])) test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) write_volume_data(core_api, test_pod_name, test_data) # Not using delete_and_wait_pod here because there is the small chance the # StatefulSet recreates the Pod quickly enough where the function won't # detect that the Pod was deleted, which will time out and throw an error. core_api.delete_namespaced_pod(name=test_pod_name, namespace='default', body=k8sclient.V1DeleteOptions()) wait_statefulset(statefulset) resp = read_volume_data(core_api, test_pod_name) assert resp == test_data
def test_statefulset_pod_deletion(core_api, storage_class, statefulset): # NOQA """ Test that a StatefulSet can spin up a new Pod with the same data after a previous Pod has been deleted. This test will only work in a CSI environment. It will automatically be disabled in FlexVolume environments. """ statefulset_name = 'statefulset-pod-deletion-test' update_test_manifests(statefulset, storage_class, statefulset_name) test_pod_name = statefulset_name + '-' + \ str(randrange(statefulset['spec']['replicas'])) test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) write_volume_data(core_api, test_pod_name, test_data) # Not using delete_and_wait_pod here because there is the small chance the # StatefulSet recreates the Pod quickly enough where the function won't # detect that the Pod was deleted, which will time out and throw an error. core_api.delete_namespaced_pod(name=test_pod_name, namespace='default', body=k8sclient.V1DeleteOptions()) wait_statefulset(statefulset) resp = read_volume_data(core_api, test_pod_name) assert resp == test_data
def test_csi_io(client, core_api, csi_pv, pvc, pod): # NOQA """ Test that input and output on a statically defined CSI volume works as expected. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ pod_name = 'csi-io-test' pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])] pvc['spec']['volumeName'] = csi_pv['metadata']['name'] test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_pv_storage(core_api, client, csi_pv, pvc) create_and_wait_pod(core_api, pod) write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) common.wait_for_volume_detached(client, csi_pv['metadata']['name']) pod_name = 'csi-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def csi_io_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA pv_name = generate_volume_name() pod_name = 'csi-io-test' create_and_wait_csi_pod_named_pv(pv_name, pod_name, client, core_api, csi_pv, pvc, pod_make, base_image, "") test_data = generate_random_data(VOLUME_RWTEST_SIZE) write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) common.wait_for_volume_detached(client, csi_pv['metadata']['name']) pod_name = 'csi-io-test-2' pod = pod_make(name=pod_name) pod['spec']['volumes'] = [ create_pvc_spec(pv_name) ] csi_pv['metadata']['name'] = pv_name csi_pv['spec']['csi']['volumeHandle'] = pv_name pvc['metadata']['name'] = pv_name pvc['spec']['volumeName'] = pv_name update_storageclass_references(CSI_PV_TEST_STORAGE_NAME, csi_pv, pvc) create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA vol_name = csi_pv['metadata']['name'] write_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = common.find_backup(client, vol_name, snap["name"]) pod2_name = 'csi-backup-test-' + str(i) create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make, base_image, b["url"]) resp = read_volume_data(core_api, pod2_name) assert resp == test_data bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap["name"]: found = True break assert not found
def test_statefulset_pod_deletion(core_api, storage_class, statefulset): # NOQA """ Test that a StatefulSet can spin up a new Pod with the same data after a previous Pod has been deleted. 1. Create a StatefulSet with VolumeClaimTemplate and Longhorn. 2. Wait for pods to run. 3. Write some data to one of the pod. 4. Delete that pod. 5. Wait for the StatefulSet to recreate the pod 6. Verify the data in the pod. """ statefulset_name = 'statefulset-pod-deletion-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) test_pod_name = statefulset_name + '-' + \ str(randrange(statefulset['spec']['replicas'])) test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) write_pod_volume_data(core_api, test_pod_name, test_data) # Not using delete_and_wait_pod here because there is the small chance the # StatefulSet recreates the Pod quickly enough where the function won't # detect that the Pod was deleted, which will time out and throw an error. core_api.delete_namespaced_pod(name=test_pod_name, namespace='default', body=k8sclient.V1DeleteOptions()) wait_statefulset(statefulset) resp = read_volume_data(core_api, test_pod_name) assert resp == test_data
def test_flexvolume_io(client, core_api, flexvolume, pod): # NOQA """ Test that input and output on a statically defined volume works as expected. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ pod_name = 'flexvolume-io-test' pod['metadata']['name'] = pod_name pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \ flexvolume['name'] pod['spec']['volumes'] = [flexvolume] test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_and_wait_pod(core_api, pod) common.write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) wait_for_volume_detached(client, flexvolume["name"]) pod_name = 'volume-driver-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA vol_name = csi_pv['metadata']['name'] write_pod_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = common.find_backup(client, vol_name, snap["name"]) pod2_name = 'csi-backup-test-' + str(i) create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make, base_image, b["url"]) resp = read_volume_data(core_api, pod2_name) assert resp == test_data bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap["name"]: found = True break assert not found
def test_provisioner_io(client, core_api, storage_class, pvc, pod): # NOQA """ Test that input and output on a StorageClass provisioned PersistentVolumeClaim works as expected. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ # Prepare pod and volume specs. pod_name = 'provisioner-io-test' pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [ create_pvc_spec(pvc['metadata']['name']) ] pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_storage(core_api, storage_class, pvc) create_and_wait_pod(core_api, pod) pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name']) write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) common.wait_for_volume_detached(client, pvc_volume_name) pod_name = 'flexvolume-provisioner-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def csi_io_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA pv_name = generate_volume_name() pod_name = 'csi-io-test' create_and_wait_csi_pod_named_pv(pv_name, pod_name, client, core_api, csi_pv, pvc, pod_make, base_image, "") test_data = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) common.wait_for_volume_detached(client, csi_pv['metadata']['name']) pod_name = 'csi-io-test-2' pod = pod_make(name=pod_name) pod['spec']['volumes'] = [ create_pvc_spec(pv_name) ] csi_pv['metadata']['name'] = pv_name csi_pv['spec']['csi']['volumeHandle'] = pv_name pvc['metadata']['name'] = pv_name pvc['spec']['volumeName'] = pv_name update_storageclass_references(CSI_PV_TEST_STORAGE_NAME, csi_pv, pvc) create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def delete_data(k8s_api_client, pod_name): file_name = 'data-' + pod_name + '.bin' test_data = generate_random_data(0) write_pod_volume_data(k8s_api_client, pod_name, test_data, filename=file_name) volume_data = read_volume_data(k8s_api_client, pod_name, filename=file_name) assert volume_data == ""
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA vol_name = csi_pv['metadata']['name'] write_pod_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = create_snapshot(client, vol_name) volume.snapshotBackup(name=snap.name) common.wait_for_backup_completion(client, vol_name, snap.name) bv, b = common.find_backup(client, vol_name, snap.name) pod2_name = 'csi-backup-test-' + str(i) create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make, base_image, b.url) resp = read_volume_data(core_api, pod2_name) assert resp == test_data delete_backup(client, bv.name, b.name)
def flexvolume_io_test(client, core_api, flexvolume, pod): # NOQA pod_name = 'flexvolume-io-test' pod['metadata']['name'] = pod_name pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \ flexvolume['name'] pod['spec']['volumes'] = [flexvolume] test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_and_wait_pod(core_api, pod) common.write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) wait_for_volume_detached(client, flexvolume["name"]) pod_name = 'volume-driver-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, vol_name, backing_image, test_data): # NOQA write_pod_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = create_snapshot(client, vol_name) volume.snapshotBackup(name=snap.name) common.wait_for_backup_completion(client, vol_name, snap.name) bv, b = common.find_backup(client, vol_name, snap.name) pod2_name = 'csi-backup-test-2' vol2_name = create_and_wait_csi_pod( pod2_name, client, core_api, csi_pv, pvc, pod_make, backing_image, b.url) volume2 = client.by_id_volume(vol2_name) resp = read_volume_data(core_api, pod2_name) assert resp == test_data delete_backup(client, bv.name, b.name) delete_and_wait_pod(core_api, pod2_name) client.delete(volume2)
def flexvolume_io_test(client, core_api, flexvolume, pod): # NOQA pod_name = 'flexvolume-io-test' pod['metadata']['name'] = pod_name pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \ flexvolume['name'] pod['spec']['volumes'] = [ flexvolume ] test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_and_wait_pod(core_api, pod) common.write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) wait_for_volume_detached(client, flexvolume["name"]) pod_name = 'volume-driver-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def test_xfs_pv(client, core_api, pod_manifest): # NOQA """ Test create PV with new XFS filesystem 1. Create a volume 2. Create a PV for the existing volume, specify `xfs` as filesystem 3. Create PVC and Pod 4. Make sure Pod is running. 5. Write data into the pod and read back for validation. Note: The volume will be formatted to XFS filesystem by Kubernetes in this case. """ volume_name = generate_volume_name() volume = create_and_check_volume(client, volume_name) create_pv_for_volume(client, core_api, volume, volume_name, "xfs") create_pvc_for_volume(client, core_api, volume, volume_name) pod_manifest['spec']['volumes'] = [{ "name": "pod-data", "persistentVolumeClaim": { "claimName": volume_name } }] pod_name = pod_manifest['metadata']['name'] create_and_wait_pod(core_api, pod_manifest) test_data = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, pod_name, test_data) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def test_statefulset_restore(client, core_api, storage_class, # NOQA statefulset): # NOQA """ Test that data can be restored into volumes usable by a StatefulSet. """ statefulset_name = 'statefulset-restore-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) pod_info = get_statefulset_pod_info(core_api, statefulset) create_and_test_backups(core_api, client, pod_info) delete_and_wait_statefulset(core_api, client, statefulset) csi = check_csi(core_api) # StatefulSet fixture already cleans these up, use the manifests instead of # the fixtures to avoid issues during teardown. pv = { 'apiVersion': 'v1', 'kind': 'PersistentVolume', 'metadata': { 'name': '' }, 'spec': { 'capacity': { 'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi) }, 'volumeMode': 'Filesystem', 'accessModes': ['ReadWriteOnce'], 'persistentVolumeReclaimPolicy': 'Delete', 'storageClassName': DEFAULT_STORAGECLASS_NAME } } pvc = { 'apiVersion': 'v1', 'kind': 'PersistentVolumeClaim', 'metadata': { 'name': '' }, 'spec': { 'accessModes': [ 'ReadWriteOnce' ], 'resources': { 'requests': { 'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi) } }, 'storageClassName': DEFAULT_STORAGECLASS_NAME } } if csi: pv['spec']['csi'] = { 'driver': 'io.rancher.longhorn', 'fsType': 'ext4', 'volumeAttributes': { 'numberOfReplicas': storage_class['parameters']['numberOfReplicas'], 'staleReplicaTimeout': storage_class['parameters']['staleReplicaTimeout'] }, 'volumeHandle': '' } else: pv['spec']['flexVolume'] = { 'driver': 'rancher.io/longhorn', 'fsType': 'ext4', 'options': { 'numberOfReplicas': storage_class['parameters']['numberOfReplicas'], 'staleReplicaTimeout': storage_class['parameters']['staleReplicaTimeout'], 'fromBackup': '', 'size': size_to_string(DEFAULT_VOLUME_SIZE * Gi) } } # Make sure that volumes still work even if the Pod and StatefulSet names # are different. for pod in pod_info: pod['pod_name'] = pod['pod_name'].replace('statefulset-restore-test', 'statefulset-restore-test-2') pod['pvc_name'] = pod['pvc_name'].replace('statefulset-restore-test', 'statefulset-restore-test-2') pv['metadata']['name'] = pod['pvc_name'] if csi: client.create_volume( name=pod['pvc_name'], size=size_to_string(DEFAULT_VOLUME_SIZE * Gi), numberOfReplicas=int( storage_class['parameters']['numberOfReplicas']), fromBackup=pod['backup_snapshot']['url']) wait_for_volume_detached(client, pod['pvc_name']) pv['spec']['csi']['volumeHandle'] = pod['pvc_name'] else: pv['spec']['flexVolume']['options']['fromBackup'] = \ pod['backup_snapshot']['url'] core_api.create_persistent_volume(pv) pvc['metadata']['name'] = pod['pvc_name'] pvc['spec']['volumeName'] = pod['pvc_name'] core_api.create_namespaced_persistent_volume_claim( body=pvc, namespace='default') statefulset_name = 'statefulset-restore-test-2' update_statefulset_manifests(statefulset, storage_class, statefulset_name) create_and_wait_statefulset(statefulset) for pod in pod_info: resp = read_volume_data(core_api, pod['pod_name']) assert resp == pod['data']
def test_csi_minimal_volume_size( client, core_api, csi_pv, pvc, pod_make): # NOQA """ Test CSI Minimal Volume Size 1. Create a PVC requesting size 5MiB. Check the PVC requested size is 5MiB and capacity size get is 10MiB. 2. Remove the PVC. 3. Create a PVC requesting size 10MiB. Check the PVC requested size and capacity size get are both 10MiB. 4. Create a pod to use this PVC. 5. Write some data to the volume and read it back to compare. """ vol_name = generate_volume_name() create_and_check_volume(client, vol_name, size=str(100*Mi)) low_storage = str(5*Mi) min_storage = str(10*Mi) pv_name = vol_name + "-pv" csi_pv['metadata']['name'] = pv_name csi_pv['spec']['csi']['volumeHandle'] = vol_name csi_pv['spec']['capacity']['storage'] = min_storage core_api.create_persistent_volume(csi_pv) pvc_name = vol_name + "-pvc" pvc['metadata']['name'] = pvc_name pvc['spec']['volumeName'] = pv_name pvc['spec']['resources']['requests']['storage'] = low_storage pvc['spec']['storageClassName'] = '' core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace='default') claim = common.wait_for_pvc_phase(core_api, pvc_name, "Bound") assert claim.spec.resources.requests['storage'] == low_storage assert claim.status.capacity['storage'] == min_storage common.delete_and_wait_pvc(core_api, pvc_name) common.delete_and_wait_pv(core_api, pv_name) wait_for_volume_detached(client, vol_name) core_api.create_persistent_volume(csi_pv) pvc['spec']['resources']['requests']['storage'] = min_storage core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace='default') claim = common.wait_for_pvc_phase(core_api, pvc_name, "Bound") assert claim.spec.resources.requests['storage'] == min_storage assert claim.status.capacity['storage'] == min_storage pod_name = vol_name + '-pod' pod = pod_make(name=pod_name) pod['spec']['volumes'] = [create_pvc_spec(pvc_name)] create_and_wait_pod(core_api, pod) test_data = "longhorn-integration-test" test_file = "test" write_pod_volume_data(core_api, pod_name, test_data, test_file) read_data = read_volume_data(core_api, pod_name, test_file) assert read_data == test_data
def test_statefulset_restore( client, core_api, storage_class, # NOQA statefulset): # NOQA """ Test that data can be restored into volumes usable by a StatefulSet. 1. Create a StatefulSet with VolumeClaimTemplate and Longhorn. 2. Wait for pods to run. 3. Create a backup for each pod. 4. Delete the StatefulSet, including the Longhorn volumes. 5. Create volumes and PV/PVC using previous backups from each Pod. 1. PVs will be created using the previous names. 2. PVCs will be created using previous name + "-2" due to statefulset has a naming policy for what should be PVC name for them. 6. Create a new StatefulSet using the previous name + "-2" 7. Wait for pods to be up. . Verify the pods contain the previous backed up data """ statefulset_name = 'statefulset-restore-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) pod_info = get_statefulset_pod_info(core_api, statefulset) create_and_test_backups(core_api, client, pod_info) delete_and_wait_statefulset(core_api, client, statefulset) csi = check_csi(core_api) # StatefulSet fixture already cleans these up, use the manifests instead of # the fixtures to avoid issues during teardown. pv = { 'apiVersion': 'v1', 'kind': 'PersistentVolume', 'metadata': { 'name': '' }, 'spec': { 'capacity': { 'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi) }, 'volumeMode': 'Filesystem', 'accessModes': ['ReadWriteOnce'], 'persistentVolumeReclaimPolicy': 'Delete', 'storageClassName': DEFAULT_STORAGECLASS_NAME } } pvc = { 'apiVersion': 'v1', 'kind': 'PersistentVolumeClaim', 'metadata': { 'name': '' }, 'spec': { 'accessModes': ['ReadWriteOnce'], 'resources': { 'requests': { 'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi) } }, 'storageClassName': DEFAULT_STORAGECLASS_NAME } } assert csi pv['spec']['csi'] = { 'driver': 'driver.longhorn.io', 'fsType': 'ext4', 'volumeAttributes': { 'numberOfReplicas': storage_class['parameters']['numberOfReplicas'], 'staleReplicaTimeout': storage_class['parameters']['staleReplicaTimeout'] }, 'volumeHandle': '' } # Make sure that volumes still work even if the Pod and StatefulSet names # are different. for pod in pod_info: pod['pod_name'] = pod['pod_name'].replace( 'statefulset-restore-test', 'statefulset-restore-test-2') pod['pvc_name'] = pod['pvc_name'].replace( 'statefulset-restore-test', 'statefulset-restore-test-2') pv['metadata']['name'] = pod['pvc_name'] client.create_volume( name=pod['pvc_name'], size=size_to_string(DEFAULT_VOLUME_SIZE * Gi), numberOfReplicas=int( storage_class['parameters']['numberOfReplicas']), fromBackup=pod['backup_snapshot']['url']) wait_for_volume_detached(client, pod['pvc_name']) pv['spec']['csi']['volumeHandle'] = pod['pvc_name'] core_api.create_persistent_volume(pv) pvc['metadata']['name'] = pod['pvc_name'] pvc['spec']['volumeName'] = pod['pvc_name'] core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace='default') statefulset_name = 'statefulset-restore-test-2' update_statefulset_manifests(statefulset, storage_class, statefulset_name) create_and_wait_statefulset(statefulset) for pod in pod_info: resp = read_volume_data(core_api, pod['pod_name']) assert resp == pod['data']
def test_upgrade(upgrade_image_tag, settings_reset, volume_name, pod_make, statefulset, storage_class): # NOQA """ Test Longhorn upgrade Prerequisite: - Disable Auto Salvage Setting 1. Find the upgrade image tag 2. Create a volume, generate and write data into the volume. 3. Create a Pod using a volume, generate and write data 4. Create a StatefulSet with 2 replicas, generate and write data to their volumes 5. Keep all volumes attached 6. Upgrade Longhorn system. 7. Check Pod and StatefulSet didn't restart after upgrade 8. Check All volumes data 9. Write data to StatefulSet pods, and Attached volume 10. Check data written to StatefulSet pods, and attached volume. 11. Detach the volume, and Delete Pod, and StatefulSet to detach theirvolumes 12. Upgrade all volumes engine images. 13. Attach the volume, and recreate Pod, and StatefulSet 14. Check All volumes data """ new_ei_name = "longhornio/longhorn-engine:" + upgrade_image_tag client = get_longhorn_api_client() core_api = get_core_api_client() host_id = get_self_host_id() pod_data_path = "/data/test" pod_volume_name = generate_volume_name() auto_salvage_setting = client.by_id_setting(SETTING_AUTO_SALVAGE) setting = client.update(auto_salvage_setting, value="false") assert setting.name == SETTING_AUTO_SALVAGE assert setting.value == "false" # Create Volume attached to a node. volume1 = create_and_check_volume(client, volume_name, size=SIZE) volume1.attach(hostId=host_id) volume1 = wait_for_volume_healthy(client, volume_name) volume1_data = write_volume_random_data(volume1) # Create Volume used by Pod pod_name, pv_name, pvc_name, pod_md5sum = \ prepare_pod_with_data_in_mb(client, core_api, pod_make, pod_volume_name, data_path=pod_data_path, add_liveness_prope=False) # Create multiple volumes used by StatefulSet statefulset_name = 'statefulset-upgrade-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) statefulset_pod_info = get_statefulset_pod_info(core_api, statefulset) for sspod_info in statefulset_pod_info: sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, sspod_info['pod_name'], sspod_info['data']) # upgrade Longhorn assert longhorn_upgrade(upgrade_image_tag) client = get_longhorn_api_client() # wait for 1 minute before checking pod restarts time.sleep(60) pod = core_api.read_namespaced_pod(name=pod_name, namespace='default') assert pod.status.container_statuses[0].restart_count == 0 for sspod_info in statefulset_pod_info: sspod = core_api.read_namespaced_pod(name=sspod_info['pod_name'], namespace='default') assert \ sspod.status.container_statuses[0].restart_count == 0 for sspod_info in statefulset_pod_info: resp = read_volume_data(core_api, sspod_info['pod_name']) assert resp == sspod_info['data'] res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path) assert res_pod_md5sum == pod_md5sum check_volume_data(volume1, volume1_data) for sspod_info in statefulset_pod_info: sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, sspod_info['pod_name'], sspod_info['data']) for sspod_info in statefulset_pod_info: resp = read_volume_data(core_api, sspod_info['pod_name']) assert resp == sspod_info['data'] volume1 = client.by_id_volume(volume_name) volume1_data = write_volume_random_data(volume1) check_volume_data(volume1, volume1_data) statefulset['spec']['replicas'] = replicas = 0 apps_api = get_apps_api_client() apps_api.patch_namespaced_stateful_set( name=statefulset_name, namespace='default', body={ 'spec': { 'replicas': replicas } }) delete_and_wait_pod(core_api, pod_name) volume = client.by_id_volume(volume_name) volume.detach() volumes = client.list_volume() for v in volumes: wait_for_volume_detached(client, v.name) engineimages = client.list_engine_image() for ei in engineimages: if ei.image == new_ei_name: new_ei = ei volumes = client.list_volume() for v in volumes: volume = client.by_id_volume(v.name) volume.engineUpgrade(image=new_ei.image) statefulset['spec']['replicas'] = replicas = 2 apps_api = get_apps_api_client() apps_api.patch_namespaced_stateful_set( name=statefulset_name, namespace='default', body={ 'spec': { 'replicas': replicas } }) wait_statefulset(statefulset) pod = pod_make(name=pod_name) pod['spec']['volumes'] = [create_pvc_spec(pvc_name)] create_and_wait_pod(core_api, pod) volume1 = client.by_id_volume(volume_name) volume1.attach(hostId=host_id) volume1 = wait_for_volume_healthy(client, volume_name) for sspod_info in statefulset_pod_info: resp = read_volume_data(core_api, sspod_info['pod_name']) assert resp == sspod_info['data'] res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path) assert res_pod_md5sum == pod_md5sum check_volume_data(volume1, volume1_data)
def test_offline_node_with_attached_volume_and_pod( client, core_api, volume_name, make_deployment_with_pvc, reset_cluster_ready_status): # NOQA """ Test offline node with attached volume and pod 1. Create PV/PVC/Deployment manifest. 2. Update deployment's tolerations to 20 seconds to speed up test 3. Update deployment's node affinity rule to avoid the current node 4. Create volume, PV/PVC and deployment. 5. Find the pod in the deployment and write `test_data` into it 6. Shutdown the node pod is running on 7. Wait for deployment to delete the pod 1. Deployment cannot delete the pod here because kubelet doesn't response 8. Force delete the terminating pod 9. Wait for the new pod to be created and the volume attached 10. Check `test_data` in the new pod """ toleration_seconds = 20 apps_api = get_apps_api_client() cloudprovider = detect_cloudprovider() volume_name = generate_volume_name() pv_name = volume_name + "-pv" pvc_name = volume_name + "-pvc" deployment_name = volume_name + "-dep" longhorn_test_node_name = get_self_host_id() deployment_manifest = make_deployment_with_pvc(deployment_name, pvc_name) unreachable_toleration = { "key": "node.kubernetes.io/unreachable", "operator": "Exists", "effect": "NoExecute", "tolerationSeconds": toleration_seconds } not_ready_toleration = { "key": "node.kubernetes.io/not-ready", "operator": "Exists", "effect": "NoExecute", "tolerationSeconds": toleration_seconds } deployment_manifest["spec"]["template"]["spec"]["tolerations"] =\ [unreachable_toleration, not_ready_toleration] node_affinity_roles = { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { "nodeSelectorTerms": [{ "matchExpressions": [{ "key": "kubernetes.io/hostname", "operator": "NotIn", "values": [longhorn_test_node_name] }] }] } } } deployment_manifest["spec"]["template"]["spec"]["affinity"] =\ node_affinity_roles longhorn_volume = create_and_check_volume(client, volume_name, size=SIZE) wait_for_volume_detached(client, volume_name) create_pv_for_volume(client, core_api, longhorn_volume, pv_name) create_pvc_for_volume(client, core_api, longhorn_volume, pvc_name) create_and_wait_deployment(apps_api, deployment_manifest) deployment_label_selector =\ "name=" + deployment_manifest["metadata"]["labels"]["name"] deployment_pod_list =\ core_api.list_namespaced_pod(namespace="default", label_selector=deployment_label_selector) assert deployment_pod_list.items.__len__() == 1 pod_name = deployment_pod_list.items[0].metadata.name test_data = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, pod_name, test_data) node_name = deployment_pod_list.items[0].spec.node_name node = cloudprovider.node_id(node_name) cloudprovider.node_shutdown(node) k8s_node_down = wait_for_node_down_k8s(node_name, core_api) assert k8s_node_down client = get_longhorn_api_client() longhorn_node_down = wait_for_node_down_longhorn(node_name, client) assert longhorn_node_down time.sleep(toleration_seconds + 5) for i in range(TERMINATING_POD_RETRYS): deployment_pod_list =\ core_api.list_namespaced_pod( namespace="default", label_selector=deployment_label_selector ) terminating_pod_name = None for pod in deployment_pod_list.items: if pod.metadata.__getattribute__("deletion_timestamp") is not None: terminating_pod_name = pod.metadata.name break if terminating_pod_name is not None: break else: time.sleep(TERMINATING_POD_INTERVAL) assert terminating_pod_name is not None core_api.delete_namespaced_pod(namespace="default", name=terminating_pod_name, grace_period_seconds=0) delete_and_wait_pod(core_api, terminating_pod_name) deployment_pod_list =\ core_api.list_namespaced_pod( namespace="default", label_selector=deployment_label_selector ) assert deployment_pod_list.items.__len__() == 1 wait_for_volume_detached(client, volume_name) wait_for_volume_healthy(client, volume_name) deployment_pod_list =\ core_api.list_namespaced_pod( namespace="default", label_selector=deployment_label_selector ) assert deployment_pod_list.items.__len__() == 1 new_pod_name = deployment_pod_list.items[0].metadata.name wait_pod(new_pod_name) resp_data = read_volume_data(core_api, new_pod_name) assert test_data == resp_data
def test_statefulset_restore( client, core_api, storage_class, # NOQA statefulset): # NOQA """ Test that data can be restored into volumes usable by a StatefulSet. """ statefulset_name = 'statefulset-restore-test' update_test_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) pod_info = get_statefulset_pod_info(core_api, statefulset) create_and_test_backups(core_api, client, pod_info) delete_and_wait_statefulset(core_api, client, statefulset) csi = True try: core_api.read_namespaced_pod(name='csi-provisioner-0', namespace='longhorn-system') except ApiException as e: if (e.status == 404): csi = False # StatefulSet fixture already cleans these up, use the manifests instead of # the fixtures to avoid issues during teardown. pv = { 'apiVersion': 'v1', 'kind': 'PersistentVolume', 'metadata': { 'name': '' }, 'spec': { 'capacity': { 'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi) }, 'volumeMode': 'Filesystem', 'accessModes': ['ReadWriteOnce'], 'persistentVolumeReclaimPolicy': 'Delete', 'storageClassName': DEFAULT_STORAGECLASS_NAME } } pvc = { 'apiVersion': 'v1', 'kind': 'PersistentVolumeClaim', 'metadata': { 'name': '' }, 'spec': { 'accessModes': ['ReadWriteOnce'], 'resources': { 'requests': { 'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi) } }, 'storageClassName': DEFAULT_STORAGECLASS_NAME } } if csi: pv['spec']['csi'] = { 'driver': 'io.rancher.longhorn', 'fsType': 'ext4', 'volumeAttributes': { 'numberOfReplicas': storage_class['parameters']['numberOfReplicas'], 'staleReplicaTimeout': storage_class['parameters']['staleReplicaTimeout'] }, 'volumeHandle': '' } else: pv['spec']['flexVolume'] = { 'driver': 'rancher.io/longhorn', 'fsType': 'ext4', 'options': { 'numberOfReplicas': storage_class['parameters']['numberOfReplicas'], 'staleReplicaTimeout': storage_class['parameters']['staleReplicaTimeout'], 'fromBackup': '', 'size': size_to_string(DEFAULT_VOLUME_SIZE * Gi) } } # Make sure that volumes still work even if the Pod and StatefulSet names # are different. for pod in pod_info: pod['pod_name'] = pod['pod_name'].replace( 'statefulset-restore-test', 'statefulset-restore-test-2') pod['pvc_name'] = pod['pvc_name'].replace( 'statefulset-restore-test', 'statefulset-restore-test-2') pv['metadata']['name'] = pod['pvc_name'] if csi: client.create_volume( name=pod['pvc_name'], size=size_to_string(DEFAULT_VOLUME_SIZE * Gi), numberOfReplicas=int( storage_class['parameters']['numberOfReplicas']), fromBackup=pod['backup_snapshot']['url']) wait_for_volume_detached(client, pod['pvc_name']) pv['spec']['csi']['volumeHandle'] = pod['pvc_name'] else: pv['spec']['flexVolume']['options']['fromBackup'] = \ pod['backup_snapshot']['url'] core_api.create_persistent_volume(pv) pvc['metadata']['name'] = pod['pvc_name'] pvc['spec']['volumeName'] = pod['pvc_name'] core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace='default') statefulset_name = 'statefulset-restore-test-2' update_test_manifests(statefulset, storage_class, statefulset_name) create_and_wait_statefulset(statefulset) for pod in pod_info: resp = read_volume_data(core_api, pod['pod_name']) assert resp == pod['data']
def test_csi_offline_expansion(client, core_api, storage_class, pvc, pod_manifest): # NOQA """ Test CSI feature: offline expansion 1. Create a new `storage_class` with `allowVolumeExpansion` set 2. Create PVC and Pod with dynamic provisioned volume from the StorageClass 3. Generate `test_data` and write to the pod 4. Delete the pod 5. Update pvc.spec.resources to expand the volume 6. Verify the volume expansion done using Longhorn API 7. Create a new pod and validate the volume content """ create_storage_class(storage_class) pod_name = 'csi-offline-expand-volume-test' pvc_name = pod_name + "-pvc" pvc['metadata']['name'] = pvc_name pvc['spec']['storageClassName'] = storage_class['metadata']['name'] create_pvc(pvc) pod_manifest['metadata']['name'] = pod_name pod_manifest['spec']['volumes'] = [{ 'name': pod_manifest['spec']['containers'][0]['volumeMounts'][0]['name'], 'persistentVolumeClaim': {'claimName': pvc_name}, }] create_and_wait_pod(core_api, pod_manifest) test_data = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) pv = wait_and_get_pv_for_pvc(core_api, pvc_name) assert pv.status.phase == "Bound" volume_name = pv.spec.csi.volume_handle wait_for_volume_detached(client, volume_name) pvc['spec']['resources'] = { 'requests': { 'storage': size_to_string(EXPANDED_VOLUME_SIZE*Gi) } } expand_and_wait_for_pvc(core_api, pvc) wait_for_volume_expansion(client, volume_name) volume = client.by_id_volume(volume_name) assert volume.state == "detached" assert volume.size == str(EXPANDED_VOLUME_SIZE*Gi) pod_manifest['metadata']['name'] = pod_name pod_manifest['spec']['volumes'] = [{ 'name': pod_manifest['spec']['containers'][0]['volumeMounts'][0]['name'], 'persistentVolumeClaim': {'claimName': pvc_name}, }] create_and_wait_pod(core_api, pod_manifest) resp = read_volume_data(core_api, pod_name) assert resp == test_data volume = client.by_id_volume(volume_name) engine = get_volume_engine(volume) assert volume.size == str(EXPANDED_VOLUME_SIZE*Gi) assert volume.size == engine.size
def test_rwx_multi_statefulset_with_same_pvc(core_api, pvc, statefulset, pod): # NOQA """ Test writing of data into a volume from multiple pods using same PVC 1. Create a volume with 'accessMode' rwx. 2. Create a PV and a PVC with access mode 'readwritemany' and attach to the volume. 3. Deploy a StatefulSet of 2 pods with the existing PVC above created. 4. Wait for both pods to come up. 5. Create a pod with the existing PVC above created. 6. Wait for StatefulSet to come up healthy. 7. Write data all three pods and compute md5sum. 8. Check the data md5sum in the share manager pod. """ pvc_name = 'pvc-multi-pods-test' statefulset_name = 'statefulset-rwx-same-pvc-test' pod_name = 'pod-rwx-same-pvc-test' pvc['metadata']['name'] = pvc_name pvc['spec']['storageClassName'] = 'longhorn' pvc['spec']['accessModes'] = ['ReadWriteMany'] core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace='default') statefulset['metadata']['name'] = \ statefulset['spec']['selector']['matchLabels']['app'] = \ statefulset['spec']['serviceName'] = \ statefulset['spec']['template']['metadata']['labels']['app'] = \ statefulset_name statefulset['spec']['template']['spec']['volumes'] = \ [create_pvc_spec(pvc_name)] del statefulset['spec']['volumeClaimTemplates'] create_and_wait_statefulset(statefulset) pv_name = get_volume_name(core_api, pvc_name) share_manager_name = 'share-manager-' + pv_name test_data = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, statefulset_name + '-0', test_data, filename='test1') assert test_data == read_volume_data(core_api, statefulset_name + '-1', filename='test1') pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc_name)] create_and_wait_pod(core_api, pod) assert test_data == read_volume_data(core_api, pod_name, filename='test1') test_data_2 = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, pod_name, test_data_2, filename='test2') command1 = 'cat /export' + '/' + pv_name + '/' + 'test1' command2 = 'cat /export' + '/' + pv_name + '/' + 'test2' assert test_data == exec_command_in_pod(core_api, command1, share_manager_name, LONGHORN_NAMESPACE) assert test_data_2 == exec_command_in_pod(core_api, command2, share_manager_name, LONGHORN_NAMESPACE)
def test_csi_expansion_with_replica_failure(client, core_api, storage_class, pvc, pod_manifest): # NOQA """ Test expansion success but with one replica expansion failure 1. Create a new `storage_class` with `allowVolumeExpansion` set 2. Create PVC and Pod with dynamic provisioned volume from the StorageClass 3. Create an empty directory with expansion snapshot tmp meta file path for one replica so that the replica expansion will fail 4. Generate `test_data` and write to the pod 5. Delete the pod and wait for volume detachment 6. Update pvc.spec.resources to expand the volume 7. Check expansion result using Longhorn API. There will be expansion error caused by the failed replica but overall the expansion should succeed. 8. Create a new pod and check if the volume will rebuild the failed replica 9. Validate the volume content, then check if data writing looks fine """ create_storage_class(storage_class) pod_name = 'csi-expansion-with-replica-failure-test' pvc_name = pod_name + "-pvc" pvc['metadata']['name'] = pvc_name pvc['spec']['storageClassName'] = storage_class['metadata']['name'] create_pvc(pvc) pod_manifest['metadata']['name'] = pod_name pod_manifest['spec']['volumes'] = [{ 'name': pod_manifest['spec']['containers'][0]['volumeMounts'][0]['name'], 'persistentVolumeClaim': {'claimName': pvc_name}, }] create_and_wait_pod(core_api, pod_manifest) expand_size = str(EXPANDED_VOLUME_SIZE*Gi) pv = wait_and_get_pv_for_pvc(core_api, pvc_name) assert pv.status.phase == "Bound" volume_name = pv.spec.csi.volume_handle volume = client.by_id_volume(volume_name) failed_replica = volume.replicas[0] fail_replica_expansion(client, core_api, volume_name, expand_size, [failed_replica]) test_data = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) wait_for_volume_detached(client, volume_name) # There will be replica expansion error info # but the expansion should succeed. pvc['spec']['resources'] = { 'requests': { 'storage': size_to_string(EXPANDED_VOLUME_SIZE*Gi) } } expand_and_wait_for_pvc(core_api, pvc) wait_for_expansion_failure(client, volume_name) wait_for_volume_expansion(client, volume_name) volume = client.by_id_volume(volume_name) assert volume.state == "detached" assert volume.size == expand_size for r in volume.replicas: if r.name == failed_replica.name: assert r.failedAt != "" else: assert r.failedAt == "" # Check if the replica will be rebuilded # and if the volume still works fine. create_and_wait_pod(core_api, pod_manifest) volume = wait_for_volume_healthy(client, volume_name) for r in volume.replicas: if r.name == failed_replica.name: assert r.mode == "" else: assert r.mode == "RW" resp = read_volume_data(core_api, pod_name) assert resp == test_data test_data = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, pod_name, test_data) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def test_recurring_jobs_when_volume_detached_unexpectedly( settings_reset, set_random_backupstore, client, core_api, apps_api, pvc, make_deployment_with_pvc): # NOQA """ Test recurring jobs when volume detached unexpectedly Context: If the volume is automatically attached by the recurring backup job, make sure that workload pod eventually is able to use the volume when volume is detached unexpectedly during the backup process. Steps: 1. Create a volume, attach to a pod of a deployment, write 500MB to the volume. 2. Scale down the deployment. The volume is detached. 3. Turn on `Allow Recurring Job While Volume Is Detached` setting. 4. Create a recurring backup job that runs every 2 mins. 5. Wait until the recurring backup job starts and the backup progress is > 50%, kill the engine process of the volume. 6. Verify volume automatically reattached and is healthy again. 7. Wait until the backup finishes. 8. Wait for the volume to be in detached state with `frontendDisabled=false` 9. Scale up the deployment. Verify that we can read the file `lost+found` from the workload pod 10. Turn off `Allow Recurring Job While Volume Is Detached` setting Clean up backups, volumes. """ recurring_job_setting = \ client.by_id_setting(SETTING_RECURRING_JOB_WHILE_VOLUME_DETACHED) client.update(recurring_job_setting, value="true") pvc_name = 'pvc-volume-detached-unexpectedly-test' pvc['metadata']['name'] = pvc_name pvc['spec']['storageClassName'] = 'longhorn' core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace='default') deployment = make_deployment_with_pvc( 'deployment-volume-detached-unexpectedly-test', pvc_name) create_and_wait_deployment(apps_api, deployment) pod_names = common.get_deployment_pod_names(core_api, deployment) vol_name = get_volume_name(core_api, pvc_name) write_pod_volume_random_data(core_api, pod_names[0], "/data/test", DATA_SIZE_IN_MB_3) data = read_volume_data(core_api, pod_names[0], 'default') deployment['spec']['replicas'] = 0 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment["metadata"]["name"]) vol = wait_for_volume_detached(client, vol_name) jobs = [{ "name": RECURRING_JOB_NAME, "cron": "*/2 * * * *", "task": "backup", "retain": 1 }] vol.recurringUpdate(jobs=jobs) time.sleep(60) wait_for_recurring_backup_to_start(client, core_api, vol_name, expected_snapshot_count=1, minimum_progress=50) crash_engine_process_with_sigkill(client, core_api, vol_name) # Check if the volume is reattached after recurring backup is interrupted time.sleep(10) wait_for_volume_healthy_no_frontend(client, vol_name) # Since the backup state is removed after the backup complete and it # could happen quickly. Checking for the both in-progress and complete # state could be hard to catch, thus we only check the complete state def backup_complete_predicate(b): return b.state == "complete" and b.error == "" common.wait_for_backup_state(client, vol_name, backup_complete_predicate) wait_for_volume_detached(client, vol_name) deployment['spec']['replicas'] = 1 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment["metadata"]["name"]) wait_deployment_replica_ready(apps_api, deployment["metadata"]["name"], 1) pod_names = common.get_deployment_pod_names(core_api, deployment) assert read_volume_data(core_api, pod_names[0], 'default') == data # Use fixture to cleanup the backupstore and since we # crashed the engine replica initiated the backup, it's # backupstore lock will still be present, so we need # to wait till the lock is expired, before we can delete # the backups vol.recurringUpdate(jobs=[]) backupstore.backupstore_wait_for_lock_expiration()