def csi_io_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA pv_name = generate_volume_name() pod_name = 'csi-io-test' create_and_wait_csi_pod_named_pv(pv_name, pod_name, client, core_api, csi_pv, pvc, pod_make, base_image, "") test_data = generate_random_data(VOLUME_RWTEST_SIZE) write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) common.wait_for_volume_detached(client, csi_pv['metadata']['name']) pod_name = 'csi-io-test-2' pod = pod_make(name=pod_name) pod['spec']['volumes'] = [create_pvc_spec(pv_name)] csi_pv['metadata']['name'] = pv_name csi_pv['spec']['csi']['volumeHandle'] = pv_name pvc['metadata']['name'] = pv_name pvc['spec']['volumeName'] = pv_name update_storageclass_references(CSI_PV_TEST_STORAGE_NAME, csi_pv, pvc) create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def test_provisioner_io(client, core_api, storage_class, pvc, pod): # NOQA """ Test that input and output on a StorageClass provisioned PersistentVolumeClaim works as expected. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ # Prepare pod and volume specs. pod_name = 'provisioner-io-test' pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [ create_pvc_spec(pvc['metadata']['name']) ] pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_storage(core_api, storage_class, pvc) create_and_wait_pod(core_api, pod) pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name']) write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) common.wait_for_volume_detached(client, pvc_volume_name) pod_name = 'flexvolume-provisioner-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA vol_name = csi_pv['metadata']['name'] write_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = common.find_backup(client, vol_name, snap["name"]) pod2_name = 'csi-backup-test-' + str(i) create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make, base_image, b["url"]) resp = read_volume_data(core_api, pod2_name) assert resp == test_data bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap["name"]: found = True break assert not found
def test_statefulset_pod_deletion(core_api, storage_class, statefulset): # NOQA """ Test that a StatefulSet can spin up a new Pod with the same data after a previous Pod has been deleted. This test will only work in a CSI environment. It will automatically be disabled in FlexVolume environments. """ statefulset_name = 'statefulset-pod-deletion-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) test_pod_name = statefulset_name + '-' + \ str(randrange(statefulset['spec']['replicas'])) test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) write_volume_data(core_api, test_pod_name, test_data) # Not using delete_and_wait_pod here because there is the small chance the # StatefulSet recreates the Pod quickly enough where the function won't # detect that the Pod was deleted, which will time out and throw an error. core_api.delete_namespaced_pod(name=test_pod_name, namespace='default', body=k8sclient.V1DeleteOptions()) wait_statefulset(statefulset) resp = read_volume_data(core_api, test_pod_name) assert resp == test_data
def test_provisioner_io(client, core_api, storage_class, pvc, pod): # NOQA """ Test that input and output on a StorageClass provisioned PersistentVolumeClaim works as expected. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ # Prepare pod and volume specs. pod_name = 'provisioner-io-test' pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])] pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_storage(core_api, storage_class, pvc) create_and_wait_pod(core_api, pod) pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name']) write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) common.wait_for_volume_detached(client, pvc_volume_name) pod_name = 'flexvolume-provisioner-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA vol_name = csi_pv['metadata']['name'] write_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = common.find_backup(client, vol_name, snap["name"]) pod2_name = 'csi-backup-test-' + str(i) create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make, base_image, b["url"]) resp = read_volume_data(core_api, pod2_name) assert resp == test_data bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap["name"]: found = True break assert not found
def test_csi_io(client, core_api, csi_pv, pvc, pod): # NOQA """ Test that input and output on a statically defined CSI volume works as expected. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ pod_name = 'csi-io-test' pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])] pvc['spec']['volumeName'] = csi_pv['metadata']['name'] test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_pv_storage(core_api, client, csi_pv, pvc) create_and_wait_pod(core_api, pod) write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) common.wait_for_volume_detached(client, csi_pv['metadata']['name']) pod_name = 'csi-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def test_statefulset_pod_deletion(core_api, storage_class, statefulset): # NOQA """ Test that a StatefulSet can spin up a new Pod with the same data after a previous Pod has been deleted. This test will only work in a CSI environment. It will automatically be disabled in FlexVolume environments. """ statefulset_name = 'statefulset-pod-deletion-test' update_test_manifests(statefulset, storage_class, statefulset_name) test_pod_name = statefulset_name + '-' + \ str(randrange(statefulset['spec']['replicas'])) test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) write_volume_data(core_api, test_pod_name, test_data) # Not using delete_and_wait_pod here because there is the small chance the # StatefulSet recreates the Pod quickly enough where the function won't # detect that the Pod was deleted, which will time out and throw an error. core_api.delete_namespaced_pod(name=test_pod_name, namespace='default', body=k8sclient.V1DeleteOptions()) wait_statefulset(statefulset) resp = read_volume_data(core_api, test_pod_name) assert resp == test_data
def test_flexvolume_io(client, core_api, flexvolume, pod): # NOQA """ Test that input and output on a statically defined volume works as expected. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ pod_name = 'flexvolume-io-test' pod['metadata']['name'] = pod_name pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \ flexvolume['name'] pod['spec']['volumes'] = [flexvolume] test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_and_wait_pod(core_api, pod) common.write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) wait_for_volume_detached(client, flexvolume["name"]) pod_name = 'volume-driver-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def csi_io_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA pv_name = generate_volume_name() pod_name = 'csi-io-test' create_and_wait_csi_pod_named_pv(pv_name, pod_name, client, core_api, csi_pv, pvc, pod_make, base_image, "") test_data = generate_random_data(VOLUME_RWTEST_SIZE) write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) common.wait_for_volume_detached(client, csi_pv['metadata']['name']) pod_name = 'csi-io-test-2' pod = pod_make(name=pod_name) pod['spec']['volumes'] = [ create_pvc_spec(pv_name) ] csi_pv['metadata']['name'] = pv_name csi_pv['spec']['csi']['volumeHandle'] = pv_name pvc['metadata']['name'] = pv_name pvc['spec']['volumeName'] = pv_name update_storageclass_references(CSI_PV_TEST_STORAGE_NAME, csi_pv, pvc) create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def create_backup(client, volname, data={}): volume = client.by_id_volume(volname) volume.snapshotCreate() if not data: data = write_volume_random_data(volume) else: data = write_volume_data(volume, data) snap = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = common.find_backup(client, volname, snap["name"]) new_b = bv.backupGet(name=b["name"]) assert new_b["name"] == b["name"] assert new_b["url"] == b["url"] assert new_b["snapshotName"] == b["snapshotName"] assert new_b["snapshotCreated"] == b["snapshotCreated"] assert new_b["created"] == b["created"] assert new_b["volumeName"] == b["volumeName"] assert new_b["volumeSize"] == b["volumeSize"] assert new_b["volumeCreated"] == b["volumeCreated"] volume = wait_for_volume_status(client, volname, "lastBackup", b["name"]) assert volume["lastBackupAt"] != "" return bv, b, snap, data
def test_statefulset_recurring_backup( client, core_api, storage_class, # NOQA statefulset): # NOQA """ Test that recurring backups on StatefulSets work properly. """ statefulset_name = 'statefulset-backup-test' update_test_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) # backup every minute job_backup = { "name": "backup", "cron": "* * * * *", "task": "backup", "retain": 2 } pod_data = get_statefulset_pod_info(core_api, statefulset) for pod in pod_data: pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE) pod['backup_snapshot'] = '' for pod in pod_data: volume = client.by_id_volume(pod['pv_name']) write_volume_data(core_api, pod['pod_name'], pod['data']) volume.recurringUpdate(jobs=[job_backup]) time.sleep(300) for pod in pod_data: volume = client.by_id_volume(pod['pv_name']) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot['removed'] is False: count += 1 # two backups + volume-head assert count == 3
def flexvolume_io_test(client, core_api, flexvolume, pod): # NOQA pod_name = 'flexvolume-io-test' pod['metadata']['name'] = pod_name pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \ flexvolume['name'] pod['spec']['volumes'] = [flexvolume] test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_and_wait_pod(core_api, pod) common.write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) wait_for_volume_detached(client, flexvolume["name"]) pod_name = 'volume-driver-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def flexvolume_io_test(client, core_api, flexvolume, pod): # NOQA pod_name = 'flexvolume-io-test' pod['metadata']['name'] = pod_name pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \ flexvolume['name'] pod['spec']['volumes'] = [ flexvolume ] test_data = generate_random_data(VOLUME_RWTEST_SIZE) create_and_wait_pod(core_api, pod) common.write_volume_data(core_api, pod_name, test_data) delete_and_wait_pod(core_api, pod_name) wait_for_volume_detached(client, flexvolume["name"]) pod_name = 'volume-driver-io-test-2' pod['metadata']['name'] = pod_name create_and_wait_pod(core_api, pod) resp = read_volume_data(core_api, pod_name) assert resp == test_data
def test_statefulset_recurring_backup(client, core_api, storage_class, # NOQA statefulset): # NOQA """ Test that recurring backups on StatefulSets work properly. """ statefulset_name = 'statefulset-backup-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) # backup every minute job_backup = {"name": "backup", "cron": "* * * * *", "task": "backup", "retain": 2} pod_data = get_statefulset_pod_info(core_api, statefulset) for pod in pod_data: pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE) pod['backup_snapshot'] = '' for pod in pod_data: volume = client.by_id_volume(pod['pv_name']) write_volume_data(core_api, pod['pod_name'], pod['data']) volume.recurringUpdate(jobs=[job_backup]) time.sleep(300) for pod in pod_data: volume = client.by_id_volume(pod['pv_name']) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot['removed'] is False: count += 1 # two backups + volume-head assert count == 3
def create_and_test_backups(api, cli, pod_info): """ Create backups for all Pods in a StatefulSet and tests that all the backups have the correct attributes. Args: api: An instance of CoreV1Api. cli: A Longhorn client instance. pod_info: A List of Pods with names and volume information. This List can be generated using the get_statefulset_pod_info function located in common.py. """ for pod in pod_info: pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE) pod['backup_snapshot'] = '' # Create backup. volume = cli.by_id_volume(pod['pv_name']) volume.snapshotCreate() write_volume_data(api, pod['pod_name'], pod['data']) pod['backup_snapshot'] = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=pod['backup_snapshot']['name']) # Wait for backup to appear. found = False for i in range(DEFAULT_BACKUP_TIMEOUT): backup_volumes = cli.list_backupVolume() for bv in backup_volumes: if bv['name'] == pod['pv_name']: found = True break if found: break time.sleep(DEFAULT_POD_INTERVAL) assert found found = False for i in range(DEFAULT_BACKUP_TIMEOUT): backups = bv.backupList() for b in backups: if b['snapshotName'] == pod['backup_snapshot']['name']: found = True break if found: break time.sleep(DEFAULT_POD_INTERVAL) assert found # Make sure backup has the correct attributes. new_b = bv.backupGet(name=b["name"]) assert new_b["name"] == b["name"] assert new_b["url"] == b["url"] assert new_b["snapshotName"] == b["snapshotName"] assert new_b["snapshotCreated"] == b["snapshotCreated"] assert new_b["created"] == b["created"] assert new_b["volumeName"] == b["volumeName"] assert new_b["volumeSize"] == b["volumeSize"] assert new_b["volumeCreated"] == b["volumeCreated"] # This backup has the url attribute we need to restore from backup. pod['backup_snapshot'] = b
def test_statefulset_backup(client, core_api, storage_class, statefulset): # NOQA """ Test that backups on StatefulSet volumes work properly. """ statefulset_name = 'statefulset-backup-test' update_test_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) pod_data = get_statefulset_pod_info(core_api, statefulset) for pod in pod_data: pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE) pod['backup_snapshot'] = '' for pod in pod_data: # Create backup. volume = client.by_id_volume(pod['pv_name']) volume.snapshotCreate() write_volume_data(core_api, pod['pod_name'], pod['data']) pod['backup_snapshot'] = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=pod['backup_snapshot']['name']) # Wait for backup to appear. found = False for i in range(100): backup_volumes = client.list_backupVolume() for bv in backup_volumes: if bv['name'] == pod['pv_name']: found = True break if found: break time.sleep(DEFAULT_POD_INTERVAL) assert found found = False for i in range(20): backups = bv.backupList() for b in backups: if b['snapshotName'] == pod['backup_snapshot']['name']: found = True break if found: break time.sleep(DEFAULT_POD_INTERVAL) assert found # Make sure backup has the correct attributes. new_b = bv.backupGet(name=b["name"]) assert new_b["name"] == b["name"] assert new_b["url"] == b["url"] assert new_b["snapshotName"] == b["snapshotName"] assert new_b["snapshotCreated"] == b["snapshotCreated"] assert new_b["created"] == b["created"] assert new_b["volumeName"] == b["volumeName"] assert new_b["volumeSize"] == b["volumeSize"] assert new_b["volumeCreated"] == b["volumeCreated"]
def create_and_test_backups(api, cli, pod_info): """ Create backups for all Pods in a StatefulSet and tests that all the backups have the correct attributes. Args: api: An instance of CoreV1Api. cli: A Longhorn client instance. pod_info: A List of Pods with names and volume information. This List can be generated using the get_statefulset_pod_info function located in common.py. """ for pod in pod_info: pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE) pod['backup_snapshot'] = '' # Create backup. volume = cli.by_id_volume(pod['pv_name']) volume.snapshotCreate() write_volume_data(api, pod['pod_name'], pod['data']) pod['backup_snapshot'] = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=pod['backup_snapshot']['name']) # Wait for backup to appear. found = False for i in range(DEFAULT_BACKUP_TIMEOUT): backup_volumes = cli.list_backupVolume() for bv in backup_volumes: if bv['name'] == pod['pv_name']: found = True break if found: break time.sleep(DEFAULT_POD_INTERVAL) assert found found = False for i in range(DEFAULT_BACKUP_TIMEOUT): backups = bv.backupList() for b in backups: if b['snapshotName'] == pod['backup_snapshot']['name']: found = True break if found: break time.sleep(DEFAULT_POD_INTERVAL) assert found # Make sure backup has the correct attributes. new_b = bv.backupGet(name=b["name"]) assert new_b["name"] == b["name"] assert new_b["url"] == b["url"] assert new_b["snapshotName"] == b["snapshotName"] assert new_b["snapshotCreated"] == b["snapshotCreated"] assert new_b["created"] == b["created"] assert new_b["volumeName"] == b["volumeName"] assert new_b["volumeSize"] == b["volumeSize"] assert new_b["volumeCreated"] == b["volumeCreated"] # This backup has the url attribute we need to restore from backup. pod['backup_snapshot'] = b
def test_recurring_jobs_for_detached_volume(set_random_backupstore, client, core_api, apps_api, volume_name, make_deployment_with_pvc): # NOQA """ Test recurring jobs for detached volume Context: In the current Longhorn implementation, users cannot do recurring backup when volumes are detached. This feature gives the users an option to do recurring backup even when volumes are detached. longhorn/longhorn#1509 Steps: 1. Change the setting allow-recurring-job-while-volume-detached to true. 2. Create and attach volume, write 50MB data to the volume. 3. Detach the volume. 4. Set the recurring backup for the volume on every minute. 5. In a 2-minutes retry loop, verify that there is exactly 1 new backup. 6. Delete the recurring backup. 7. Create a PV and PVC from the volume. 8. Create a deployment of 1 pod using the PVC. 9. Write 400MB data to the volume from the pod. 10. Scale down the deployment. Wait until the volume is detached. 11. Set the recurring backup for every 2 minutes. 12. Wait util the recurring backup starts, scale up the deployment to 1 pod. 13. Verify that during the recurring backup, the volume's frontend is disabled, and pod cannot start. 14. Wait for the recurring backup finishes. Delete the recurring backup. 15. In a 10-minutes retry loop, verify that the pod can eventually start. 16. Change the setting allow-recurring-job-while-volume-detached to false. 17. Cleanup. """ recurring_job_setting = \ client.by_id_setting(SETTING_RECURRING_JOB_WHILE_VOLUME_DETACHED) client.update(recurring_job_setting, value="true") vol = common.create_and_check_volume(client, volume_name, size=str(1 * Gi)) lht_hostId = get_self_host_id() vol.attach(hostId=lht_hostId) vol = wait_for_volume_healthy(client, vol.name) data = { 'pos': 0, 'content': common.generate_random_data(50 * Mi), } common.write_volume_data(vol, data) # Give sometimes for data to flush to disk time.sleep(15) vol.detach(hostId="") vol = common.wait_for_volume_detached(client, vol.name) jobs = [{ "name": RECURRING_JOB_NAME, "cron": "*/1 * * * *", "task": "backup", "retain": 1 }] vol.recurringUpdate(jobs=jobs) common.wait_for_backup_completion(client, vol.name) for _ in range(4): bv = client.by_id_backupVolume(vol.name) backups = bv.backupList().data assert len(backups) == 1 time.sleep(30) vol.recurringUpdate(jobs=[]) pv_name = volume_name + "-pv" common.create_pv_for_volume(client, core_api, vol, pv_name) pvc_name = volume_name + "-pvc" common.create_pvc_for_volume(client, core_api, vol, pvc_name) deployment_name = volume_name + "-dep" deployment = make_deployment_with_pvc(deployment_name, pvc_name) common.create_and_wait_deployment(apps_api, deployment) size_mb = 400 pod_names = common.get_deployment_pod_names(core_api, deployment) write_pod_volume_random_data(core_api, pod_names[0], "/data/test", size_mb) deployment['spec']['replicas'] = 0 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment["metadata"]["name"]) vol = common.wait_for_volume_detached(client, vol.name) jobs = [{ "name": RECURRING_JOB_NAME, "cron": "*/2 * * * *", "task": "backup", "retain": 1 }] vol.recurringUpdate(jobs=jobs) common.wait_for_backup_to_start(client, vol.name) deployment['spec']['replicas'] = 1 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment["metadata"]["name"]) deployment_label_name = deployment["metadata"]["labels"]["name"] common.wait_pod_auto_attach_after_first_backup_completion( client, core_api, vol.name, deployment_label_name) vol.recurringUpdate(jobs=[]) pod_names = common.get_deployment_pod_names(core_api, deployment) common.wait_for_pod_phase(core_api, pod_names[0], pod_phase="Running")