def test_recurring_job_in_storageclass(set_random_backupstore, client, core_api, storage_class, statefulset): # NOQA """ Test create volume with StorageClass contains recurring jobs 1. Create a StorageClass with recurring jobs 2. Create a StatefulSet with PVC template and StorageClass 3. Verify the recurring jobs run correctly. """ statefulset_name = 'recurring-job-in-storageclass-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) storage_class["parameters"]["recurringJobs"] = json.dumps(create_jobs1()) create_storage_class(storage_class) # wait until the beginning of an even minute wait_until_begin_of_an_even_minute() start_time = datetime.utcnow() create_and_wait_statefulset(statefulset) statefulset_creating_duration = datetime.utcnow() - start_time assert 150 > statefulset_creating_duration.seconds # We want to write data exactly at the 150th second since the start_time time.sleep(150 - statefulset_creating_duration.seconds) pod_info = get_statefulset_pod_info(core_api, statefulset) volume_info = [p['pv_name'] for p in pod_info] pod_names = [p['pod_name'] for p in pod_info] # write random data to volume to trigger recurring snapshot and backup job volume_data_path = "/data/test" for pod_name in pod_names: write_pod_volume_random_data(core_api, pod_name, volume_data_path, 2) time.sleep(150) # 2.5 minutes for volume_name in volume_info: # NOQA volume = client.by_id_volume(volume_name) check_jobs1_result(volume)
def test_allow_volume_creation_with_degraded_availability_csi( client, core_api, apps_api, make_deployment_with_pvc): # NOQA """ Test Allow Volume Creation with Degraded Availability (CSI) Requirement: 1. Set `allow-volume-creation-with-degraded-availability` to true. 2. Set `node-level-soft-anti-affinity` to false. Steps: 1. Disable scheduling for node 3. 2. Create a Deployment Pod with a volume and 3 replicas. 1. After the volume is attached, scheduling error should be seen. 3. Write data to the Pod. 4. Scale down the deployment to 0 to detach the volume. 1. Scheduled condition should become true. 5. Scale up the deployment back to 1 and verify the data. 1. Scheduled condition should become false. 6. Enable the scheduling for node 3. 1. Volume should start rebuilding on the node 3 soon. 2. Once the rebuilding starts, the scheduled condition should become true. 7. Once rebuild finished, scale down and back the deployment to verify the data. """ setting = client.by_id_setting(common.SETTING_DEGRADED_AVAILABILITY) client.update(setting, value="true") setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY) client.update(setting, value="false") nodes = client.list_node() node3 = nodes[2] client.update(node3, allowScheduling=False) vol = common.create_and_check_volume(client, generate_volume_name(), size=str(500 * Mi)) pv_name = vol.name + "-pv" common.create_pv_for_volume(client, core_api, vol, pv_name) pvc_name = vol.name + "-pvc" common.create_pvc_for_volume(client, core_api, vol, pvc_name) deployment_name = vol.name + "-dep" deployment = make_deployment_with_pvc(deployment_name, pvc_name) deployment["spec"]["replicas"] = 3 apps_api.create_namespaced_deployment(body=deployment, namespace='default') common.wait_for_volume_status(client, vol.name, common.VOLUME_FIELD_STATE, common.VOLUME_STATE_ATTACHED) common.wait_scheduling_failure(client, vol.name) data_path = "/data/test" pod = common.wait_and_get_any_deployment_pod(core_api, deployment_name) common.write_pod_volume_random_data(core_api, pod.metadata.name, data_path, common.DATA_SIZE_IN_MB_2) created_md5sum = get_pod_data_md5sum(core_api, pod.metadata.name, data_path) deployment['spec']['replicas'] = 0 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment_name) vol = common.wait_for_volume_detached(client, vol.name) assert vol.conditions[VOLUME_CONDITION_SCHEDULED]['status'] == "True" deployment['spec']['replicas'] = 1 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment_name) common.wait_for_volume_status(client, vol.name, common.VOLUME_FIELD_STATE, common.VOLUME_STATE_ATTACHED) common.wait_for_volume_condition_scheduled(client, vol.name, "status", common.CONDITION_STATUS_FALSE) pod = common.wait_and_get_any_deployment_pod(core_api, deployment_name) assert created_md5sum == get_pod_data_md5sum(core_api, pod.metadata.name, data_path) client.update(node3, allowScheduling=True) common.wait_for_rebuild_start(client, vol.name) vol = client.by_id_volume(vol.name) assert vol.conditions[VOLUME_CONDITION_SCHEDULED]['status'] == "True" common.wait_for_rebuild_complete(client, vol.name) deployment['spec']['replicas'] = 0 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment_name) common.wait_for_volume_detached(client, vol.name) deployment['spec']['replicas'] = 1 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment_name) common.wait_for_volume_status(client, vol.name, common.VOLUME_FIELD_STATE, common.VOLUME_STATE_ATTACHED) pod = common.wait_and_get_any_deployment_pod(core_api, deployment_name) assert created_md5sum == get_pod_data_md5sum(core_api, pod.metadata.name, data_path)
def write_data_into_pod(pod_name_and_data_path): pod_info = pod_name_and_data_path.split(':') core_api = get_core_api_client() # NOQA write_pod_volume_random_data(core_api, pod_info[0], pod_info[1], DATA_SIZE_IN_MB_3)
def test_engine_live_upgrade_with_intensive_data_writing( client, core_api, volume_name, pod_make): # NOQA """ Test engine live upgrade with intensive data writing 1. Deploy a compatible new engine image 2. Create a volume(with the old default engine image) with /PV/PVC/Pod and wait for pod to be deployed. 3. Write data to a tmp file in the pod and get the md5sum 4. Upgrade the volume to the new engine image without waiting. 5. Keep copying data from the tmp file to the volume during the live upgrade. 6. Wait until the upgrade completed, verify the volume engine image changed 7. Wait for new replica mode update then check the engine status. 8. Verify all engine and replicas' engine image changed 9. Verify the reference count of the new engine image changed 10. Check the existing data. Then write new data to the upgraded volume and get the md5sum. 11. Delete the pod and wait for the volume detached. Then check engine and replicas's engine image again. 12. Recreate the pod. 13. Check if the attached volume is state `healthy` rather than `degraded`. 14. Check the data. """ default_img = common.get_default_engine_image(client) default_img_name = default_img.name default_img = wait_for_engine_image_ref_count(client, default_img_name, 0) cli_v = default_img.cliAPIVersion cli_minv = default_img.cliAPIMinVersion ctl_v = default_img.controllerAPIVersion ctl_minv = default_img.controllerAPIMinVersion data_v = default_img.dataFormatVersion data_minv = default_img.dataFormatMinVersion engine_upgrade_image = common.get_upgrade_test_image( cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv) new_img = client.create_engine_image(image=engine_upgrade_image) new_img_name = new_img.name ei_status_value = get_engine_image_status_value(client, new_img_name) new_img = wait_for_engine_image_state(client, new_img_name, ei_status_value) assert new_img.refCount == 0 assert new_img.noRefSince != "" default_img = common.get_default_engine_image(client) default_img_name = default_img.name pod_name = volume_name + "-pod" pv_name = volume_name + "-pv" pvc_name = volume_name + "-pvc" pod = pod_make(name=pod_name) volume = create_and_check_volume(client, volume_name, num_of_replicas=3, size=str(1 * Gi)) original_engine_image = volume.engineImage assert original_engine_image != engine_upgrade_image create_pv_for_volume(client, core_api, volume, pv_name) create_pvc_for_volume(client, core_api, volume, pvc_name) pod['spec']['volumes'] = [create_pvc_spec(pvc_name)] create_and_wait_pod(core_api, pod) volume = client.by_id_volume(volume_name) assert volume.engineImage == original_engine_image assert volume.currentImage == original_engine_image engine = get_volume_engine(volume) assert engine.engineImage == original_engine_image assert engine.currentImage == original_engine_image for replica in volume.replicas: assert replica.engineImage == original_engine_image assert replica.currentImage == original_engine_image data_path0 = "/tmp/test" data_path1 = "/data/test1" write_pod_volume_random_data(core_api, pod_name, data_path0, RANDOM_DATA_SIZE_LARGE) original_md5sum1 = get_pod_data_md5sum(core_api, pod_name, data_path0) volume.engineUpgrade(image=engine_upgrade_image) # Keep writing data to the volume during the live upgrade copy_pod_volume_data(core_api, pod_name, data_path0, data_path1) # Wait for live upgrade complete wait_for_volume_current_image(client, volume_name, engine_upgrade_image) volume = wait_for_volume_replicas_mode(client, volume_name, "RW") engine = get_volume_engine(volume) assert engine.engineImage == engine_upgrade_image check_volume_endpoint(volume) wait_for_engine_image_ref_count(client, default_img_name, 0) wait_for_engine_image_ref_count(client, new_img_name, 1) volume_file_md5sum1 = get_pod_data_md5sum(core_api, pod_name, data_path1) assert volume_file_md5sum1 == original_md5sum1 data_path2 = "/data/test2" write_pod_volume_random_data(core_api, pod_name, data_path2, RANDOM_DATA_SIZE_SMALL) original_md5sum2 = get_pod_data_md5sum(core_api, pod_name, data_path2) delete_and_wait_pod(core_api, pod_name) volume = wait_for_volume_detached(client, volume_name) assert len(volume.replicas) == 3 assert volume.engineImage == engine_upgrade_image engine = get_volume_engine(volume) assert engine.engineImage == engine_upgrade_image for replica in volume.replicas: assert replica.engineImage == engine_upgrade_image create_and_wait_pod(core_api, pod) common.wait_for_volume_healthy(client, volume_name) volume_file_md5sum1 = get_pod_data_md5sum(core_api, pod_name, data_path1) assert volume_file_md5sum1 == original_md5sum1 volume_file_md5sum2 = get_pod_data_md5sum(core_api, pod_name, data_path2) assert volume_file_md5sum2 == original_md5sum2
def test_recurring_jobs_when_volume_detached_unexpectedly( settings_reset, set_random_backupstore, client, core_api, apps_api, pvc, make_deployment_with_pvc): # NOQA """ Test recurring jobs when volume detached unexpectedly Context: If the volume is automatically attached by the recurring backup job, make sure that workload pod eventually is able to use the volume when volume is detached unexpectedly during the backup process. Steps: 1. Create a volume, attach to a pod of a deployment, write 500MB to the volume. 2. Scale down the deployment. The volume is detached. 3. Turn on `Allow Recurring Job While Volume Is Detached` setting. 4. Create a recurring backup job that runs every 2 mins. 5. Wait until the recurring backup job starts and the backup progress is > 50%, kill the engine process of the volume. 6. Verify volume automatically reattached and is healthy again. 7. Wait until the backup finishes. 8. Wait for the volume to be in detached state with `frontendDisabled=false` 9. Scale up the deployment. Verify that we can read the file `lost+found` from the workload pod 10. Turn off `Allow Recurring Job While Volume Is Detached` setting Clean up backups, volumes. """ recurring_job_setting = \ client.by_id_setting(SETTING_RECURRING_JOB_WHILE_VOLUME_DETACHED) client.update(recurring_job_setting, value="true") pvc_name = 'pvc-volume-detached-unexpectedly-test' pvc['metadata']['name'] = pvc_name pvc['spec']['storageClassName'] = 'longhorn' core_api.create_namespaced_persistent_volume_claim(body=pvc, namespace='default') deployment = make_deployment_with_pvc( 'deployment-volume-detached-unexpectedly-test', pvc_name) create_and_wait_deployment(apps_api, deployment) pod_names = common.get_deployment_pod_names(core_api, deployment) vol_name = get_volume_name(core_api, pvc_name) write_pod_volume_random_data(core_api, pod_names[0], "/data/test", DATA_SIZE_IN_MB_3) data = read_volume_data(core_api, pod_names[0], 'default') deployment['spec']['replicas'] = 0 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment["metadata"]["name"]) vol = wait_for_volume_detached(client, vol_name) jobs = [{ "name": RECURRING_JOB_NAME, "cron": "*/2 * * * *", "task": "backup", "retain": 1 }] vol.recurringUpdate(jobs=jobs) time.sleep(60) wait_for_recurring_backup_to_start(client, core_api, vol_name, expected_snapshot_count=1, minimum_progress=50) crash_engine_process_with_sigkill(client, core_api, vol_name) # Check if the volume is reattached after recurring backup is interrupted time.sleep(10) wait_for_volume_healthy_no_frontend(client, vol_name) # Since the backup state is removed after the backup complete and it # could happen quickly. Checking for the both in-progress and complete # state could be hard to catch, thus we only check the complete state def backup_complete_predicate(b): return b.state == "complete" and b.error == "" common.wait_for_backup_state(client, vol_name, backup_complete_predicate) wait_for_volume_detached(client, vol_name) deployment['spec']['replicas'] = 1 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment["metadata"]["name"]) wait_deployment_replica_ready(apps_api, deployment["metadata"]["name"], 1) pod_names = common.get_deployment_pod_names(core_api, deployment) assert read_volume_data(core_api, pod_names[0], 'default') == data # Use fixture to cleanup the backupstore and since we # crashed the engine replica initiated the backup, it's # backupstore lock will still be present, so we need # to wait till the lock is expired, before we can delete # the backups vol.recurringUpdate(jobs=[]) backupstore.backupstore_wait_for_lock_expiration()
def test_recurring_jobs_for_detached_volume(set_random_backupstore, client, core_api, apps_api, volume_name, make_deployment_with_pvc): # NOQA """ Test recurring jobs for detached volume Context: In the current Longhorn implementation, users cannot do recurring backup when volumes are detached. This feature gives the users an option to do recurring backup even when volumes are detached. longhorn/longhorn#1509 Steps: 1. Change the setting allow-recurring-job-while-volume-detached to true. 2. Create and attach volume, write 50MB data to the volume. 3. Detach the volume. 4. Set the recurring backup for the volume on every minute. 5. In a 2-minutes retry loop, verify that there is exactly 1 new backup. 6. Delete the recurring backup. 7. Create a PV and PVC from the volume. 8. Create a deployment of 1 pod using the PVC. 9. Write 400MB data to the volume from the pod. 10. Scale down the deployment. Wait until the volume is detached. 11. Set the recurring backup for every 2 minutes. 12. Wait util the recurring backup starts, scale up the deployment to 1 pod. 13. Verify that during the recurring backup, the volume's frontend is disabled, and pod cannot start. 14. Wait for the recurring backup finishes. Delete the recurring backup. 15. In a 10-minutes retry loop, verify that the pod can eventually start. 16. Change the setting allow-recurring-job-while-volume-detached to false. 17. Cleanup. """ recurring_job_setting = \ client.by_id_setting(SETTING_RECURRING_JOB_WHILE_VOLUME_DETACHED) client.update(recurring_job_setting, value="true") vol = common.create_and_check_volume(client, volume_name, size=str(1 * Gi)) lht_hostId = get_self_host_id() vol.attach(hostId=lht_hostId) vol = wait_for_volume_healthy(client, vol.name) data = { 'pos': 0, 'content': common.generate_random_data(50 * Mi), } common.write_volume_data(vol, data) # Give sometimes for data to flush to disk time.sleep(15) vol.detach(hostId="") vol = common.wait_for_volume_detached(client, vol.name) jobs = [{ "name": RECURRING_JOB_NAME, "cron": "*/1 * * * *", "task": "backup", "retain": 1 }] vol.recurringUpdate(jobs=jobs) common.wait_for_backup_completion(client, vol.name) for _ in range(4): bv = client.by_id_backupVolume(vol.name) backups = bv.backupList().data assert len(backups) == 1 time.sleep(30) vol.recurringUpdate(jobs=[]) pv_name = volume_name + "-pv" common.create_pv_for_volume(client, core_api, vol, pv_name) pvc_name = volume_name + "-pvc" common.create_pvc_for_volume(client, core_api, vol, pvc_name) deployment_name = volume_name + "-dep" deployment = make_deployment_with_pvc(deployment_name, pvc_name) common.create_and_wait_deployment(apps_api, deployment) size_mb = 400 pod_names = common.get_deployment_pod_names(core_api, deployment) write_pod_volume_random_data(core_api, pod_names[0], "/data/test", size_mb) deployment['spec']['replicas'] = 0 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment["metadata"]["name"]) vol = common.wait_for_volume_detached(client, vol.name) jobs = [{ "name": RECURRING_JOB_NAME, "cron": "*/2 * * * *", "task": "backup", "retain": 1 }] vol.recurringUpdate(jobs=jobs) common.wait_for_backup_to_start(client, vol.name) deployment['spec']['replicas'] = 1 apps_api.patch_namespaced_deployment(body=deployment, namespace='default', name=deployment["metadata"]["name"]) deployment_label_name = deployment["metadata"]["labels"]["name"] common.wait_pod_auto_attach_after_first_backup_completion( client, core_api, vol.name, deployment_label_name) vol.recurringUpdate(jobs=[]) pod_names = common.get_deployment_pod_names(core_api, deployment) common.wait_for_pod_phase(core_api, pod_names[0], pod_phase="Running")
def test_cloning_basic(client, core_api, pvc, pod, clone_pvc, clone_pod, storage_class_name='longhorn'): # NOQA """ 1. Create a PVC: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: source-pvc spec: storageClassName: longhorn accessModes: - ReadWriteOnce resources: requests: storage: 3Gi ``` 2. Specify the `source-pvc` in a pod yaml and start the pod 3. Wait for the pod to be running, write some data to the mount path of the volume 4. Clone a volume by creating the PVC: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: cloned-pvc spec: storageClassName: longhorn dataSource: name: source-pvc kind: PersistentVolumeClaim accessModes: - ReadWriteOnce resources: requests: storage: 3Gi ``` 5. Wait for the `CloneStatus.State` in `cloned-pvc` to be `completed` 6. Clone volume should get detached after cloning completion, wait for it. 7. Specify the `cloned-pvc` in a cloned pod yaml and deploy the cloned pod 8. In 3-min retry loop, wait for the cloned pod to be running 9. Verify the data in `cloned-pvc` is the same as in `source-pvc` 10. In 2-min retry loop, verify the volume of the `clone-pvc` eventually becomes healthy """ # Step-1 source_pvc_name = 'source-pvc' + generate_random_suffix() pvc['metadata']['name'] = source_pvc_name pvc['spec']['storageClassName'] = storage_class_name core_api.create_namespaced_persistent_volume_claim( body=pvc, namespace='default') wait_for_pvc_phase(core_api, source_pvc_name, "Bound") # Step-2 pod_name = 'source-pod' + generate_random_suffix() pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(source_pvc_name)] create_and_wait_pod(core_api, pod) # Step-3 write_pod_volume_random_data(core_api, pod_name, '/data/test', DATA_SIZE_IN_MB_2) source_data = get_pod_data_md5sum(core_api, pod_name, '/data/test') # Step-4 clone_pvc_name = 'clone-pvc' + generate_random_suffix() clone_pvc['metadata']['name'] = clone_pvc_name clone_pvc['spec']['storageClassName'] = storage_class_name clone_pvc['spec']['dataSource'] = { 'name': source_pvc_name, 'kind': 'PersistentVolumeClaim' } core_api.create_namespaced_persistent_volume_claim( body=clone_pvc, namespace='default') wait_for_pvc_phase(core_api, clone_pvc_name, "Bound") # Step-5 clone_volume_name = get_volume_name(core_api, clone_pvc_name) wait_for_volume_clone_status(client, clone_volume_name, VOLUME_FIELD_STATE, VOLUME_FIELD_CLONE_COMPLETED) # Step-6 wait_for_volume_detached(client, clone_volume_name) # Step-7,8 clone_pod_name = 'clone-pod' + generate_random_suffix() clone_pod['metadata']['name'] = clone_pod_name clone_pod['spec']['volumes'] = [create_pvc_spec(clone_pvc_name)] create_and_wait_pod(core_api, clone_pod) clone_data = get_pod_data_md5sum(core_api, clone_pod_name, '/data/test') # Step-9 assert source_data == clone_data # Step-10 wait_for_volume_healthy(client, clone_volume_name)
def test_cloning_interrupted(client, core_api, pvc, pod, clone_pvc, clone_pod): # NOQA """ 1. Create a PVC: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: source-pvc spec: storageClassName: longhorn accessModes: - ReadWriteOnce resources: requests: storage: 3Gi ``` 2. Specify the `source-pvc` in a pod yaml and start the pod 3. Wait for the pod to be running, write 500MB of data to the mount path of the volume 4. Clone a volume by creating the PVC: ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: cloned-pvc spec: storageClassName: longhorn dataSource: name: source-pvc kind: PersistentVolumeClaim accessModes: - ReadWriteOnce resources: requests: storage: 3Gi ``` 5. Wait for the `CloneStatus.State` in `cloned-pvc` to be `initiated` 6. Kill all replicas process of the `source-pvc` 7. Wait for the `CloneStatus.State` in `cloned-pvc` to be `failed` 8. Clean up `clone-pvc` 9. Redeploy `cloned-pvc` and clone pod 10. In 3-min retry loop, verify cloned pod become running 11. `cloned-pvc` has the same data as `source-pvc` 12. In 2-min retry loop, verify the volume of the `clone-pvc` eventually becomes healthy. """ # Step-1 source_pvc_name = 'source-pvc' + generate_random_suffix() pvc['metadata']['name'] = source_pvc_name pvc['spec']['storageClassName'] = 'longhorn' core_api.create_namespaced_persistent_volume_claim( body=pvc, namespace='default') wait_for_pvc_phase(core_api, source_pvc_name, "Bound") # Step-2 pod_name = 'source-pod' + generate_random_suffix() pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(source_pvc_name)] create_and_wait_pod(core_api, pod) # Step-3 write_pod_volume_random_data(core_api, pod_name, '/data/test', DATA_SIZE_IN_MB_3) source_data = get_pod_data_md5sum(core_api, pod_name, '/data/test') source_volume_name = get_volume_name(core_api, source_pvc_name) # Step-4 clone_pvc_name = 'clone-pvc' + generate_random_suffix() clone_pvc['metadata']['name'] = clone_pvc_name clone_pvc['spec']['storageClassName'] = 'longhorn' clone_pvc['spec']['dataSource'] = { 'name': source_pvc_name, 'kind': 'PersistentVolumeClaim' } core_api.create_namespaced_persistent_volume_claim( body=clone_pvc, namespace='default') # Step-5 clone_volume_name = get_clone_volume_name(client, source_volume_name) wait_for_volume_clone_status(client, clone_volume_name, VOLUME_FIELD_STATE, 'initiated') # Step-6 crash_replica_processes(client, core_api, source_volume_name) # Step-7 wait_for_volume_faulted(client, source_volume_name) wait_for_volume_clone_status(client, clone_volume_name, VOLUME_FIELD_STATE, 'failed') # Step-8 delete_and_wait_pvc(core_api, clone_pvc_name) # Step-9 clone_pvc_name = 'clone-pvc-2' + generate_random_suffix() clone_pvc['metadata']['name'] = clone_pvc_name clone_pvc['spec']['storageClassName'] = 'longhorn' clone_pvc['spec']['dataSource'] = { 'name': source_pvc_name, 'kind': 'PersistentVolumeClaim' } core_api.create_namespaced_persistent_volume_claim( body=clone_pvc, namespace='default') wait_for_pvc_phase(core_api, clone_pvc_name, "Bound") # Step-9 clone_pod_name = 'clone-pod' + generate_random_suffix() clone_pod['metadata']['name'] = clone_pod_name clone_pod['spec']['volumes'] = [create_pvc_spec(clone_pvc_name)] create_and_wait_pod(core_api, clone_pod) # Step-10 clone_volume_name = get_volume_name(core_api, clone_pvc_name) wait_for_volume_clone_status(client, clone_volume_name, VOLUME_FIELD_STATE, VOLUME_FIELD_CLONE_COMPLETED) # Step-11 clone_data = get_pod_data_md5sum(core_api, clone_pod_name, '/data/test') assert source_data == clone_data # Step-12 wait_for_volume_healthy(client, clone_volume_name)
def test_data_locality_basic(client, core_api, volume_name, pod, settings_reset): # NOQA """ Test data locality basic feature Context: Data Locality feature allows users to have an option to keep a local replica on the same node as the consuming pod. Longhorn is currently supporting 2 modes: - disabled: Longhorn does not try to keep a local replica - best-effort: Longhorn try to keep a local replica See manual tests at: https://github.com/longhorn/longhorn/issues/1045#issuecomment-680706283 Steps: Case 1: Test that Longhorn builds a local replica on the engine node 1. Create a volume(1) with 1 replica and dataLocality set to disabled 2. Find node where the replica is located on. Let's call the node is replica-node 3. Attach the volume to a node different than replica-node. Let call the node is engine-node 4. Write 200MB data to volume(1) 5. Use a retry loop to verify that Longhorn does not create a replica on the engine-node 6. Update dataLocality to best-effort for volume(1) 7. Use a retry loop to verify that Longhorn creates and rebuilds a replica on the engine-node and remove the other replica 8. detach the volume(1) and attach it to a different node. Let's call the new node is new-engine-node and the old node is old-engine-node 9. Wait for volume(1) to finish attaching 10. Use a retry loop to verify that Longhorn creates and rebuilds a replica on the new-engine-node and remove the replica on old-engine-node Case 2: Test that Longhorn prioritizes deleting replicas on the same node 1. Add the tag AVAIL to node-1 and node-2 2. Set node soft anti-affinity to `true`. 3. Create a volume(2) with 3 replicas and dataLocality set to best-effort 4. Use a retry loop to verify that all 3 replicas are on node-1 and node-2, no replica is on node-3 5. Attach volume(2) to node-3 6. User a retry loop to verify that there is no replica on node-3 and we can still read/write to volume(2) 7. Find the node which contains 2 replicas. Let call the node is most-replica-node 8. Set the replica count to 2 for volume(2) 9. Verify that Longhorn remove one replica from most-replica-node Case 3: Test that the volume is not corrupted if there is an unexpected detachment during building local replica 1. Remove the tag AVAIL from node-1 and node-2 Set node soft anti-affinity to `false`. 2. Create a volume(3) with 1 replicas and dataLocality set to best-effort 3. Attach volume(3) to node-3. 4. Use a retry loop to verify that volume(3) has only 1 replica on node-3 5. Write 800MB data to volume(3) 6. Detach volume(3) 7. Attach volume(3) to node-1 8. Use a retry loop to: Wait until volume(3) finishes attaching. Wait until Longhorn start rebuilding a replica on node-1 Immediately detach volume(3) 9. Verify that the replica on node-1 is in ERR state. 10. Attach volume(3) to node-1 11. Wait until volume(3) finishes attaching. 12. Use a retry loop to verify the Longhorn cleanup the ERR replica, rebuild a new replica on node-1, and remove the replica on node-3 Case 4: Make sure failed to schedule local replica doesn't block the the creation of other replicas. 1. Disable scheduling for node-3 2. Create a vol with 1 replica, `dataLocality = best-effort`. The replica is scheduled on a node (say node-1) 3. Attach vol to node-3. There is a fail-to-schedule replica with Spec.HardNodeAffinity=node-3 4. Increase numberOfReplica to 3. Verify that the replica set contains: one on node-1, one on node-2, one failed replica with Spec.HardNodeAffinity=node-3. 5. Decrease numberOfReplica to 2. Verify that the replica set contains: one on node-1, one on node-2, one failed replica with Spec.HardNodeAffinity=node-3. 6. Decrease numberOfReplica to 1. Verify that the replica set contains: one on node-1 or node-2, one failed replica with Spec.HardNodeAffinity=node-3. 7. Decrease numberOfReplica to 2. Verify that the replica set contains: one on node-1, one on node-2, one failed replica with Spec.HardNodeAffinity=node-3. 8. Turn off data locality by set `dataLocality=disabled` for the vol. Verify that the replica set contains: one on node-1, one on node-2 9. clean up """ # Case 1: Test that Longhorn builds a local replica on the engine node nodes = client.list_node() default_data_locality_setting = \ client.by_id_setting(SETTING_DEFAULT_DATA_LOCALITY) try: client.update(default_data_locality_setting, value="disabled") except Exception as e: print("Exception when update Default Data Locality setting", default_data_locality_setting, e) volume1_name = volume_name + "-1" volume1_size = str(500 * Mi) volume1_data_path = "/data/test" pv1_name = volume1_name + "-pv" pvc1_name = volume1_name + "-pvc" pod1_name = volume1_name + "-pod" pod1 = pod pod1['metadata']['name'] = pod1_name volume1 = create_and_check_volume(client, volume1_name, num_of_replicas=1, size=volume1_size) volume1 = client.by_id_volume(volume1_name) create_pv_for_volume(client, core_api, volume1, pv1_name) create_pvc_for_volume(client, core_api, volume1, pvc1_name) volume1 = client.by_id_volume(volume1_name) volume1_replica_node = volume1.replicas[0]['hostId'] volume1_attached_node = None for node in nodes: if node.name != volume1_replica_node: volume1_attached_node = node.name break assert volume1_attached_node is not None pod1['spec']['volumes'] = [{ "name": "pod-data", "persistentVolumeClaim": { "claimName": pvc1_name } }] pod1['spec']['nodeSelector'] = \ {"kubernetes.io/hostname": volume1_attached_node} create_and_wait_pod(core_api, pod1) write_pod_volume_random_data(core_api, pod1_name, volume1_data_path, DATA_SIZE_IN_MB_2) for i in range(10): volume1 = client.by_id_volume(volume1_name) assert len(volume1.replicas) == 1 assert volume1.replicas[0]['hostId'] != volume1_attached_node time.sleep(1) volume1 = client.by_id_volume(volume1_name) volume1.updateDataLocality(dataLocality="best-effort") for _ in range(RETRY_COUNTS): volume1 = client.by_id_volume(volume1_name) assert volume1[VOLUME_FIELD_ROBUSTNESS] == VOLUME_ROBUSTNESS_HEALTHY if len(volume1.replicas) == 1 and \ volume1.replicas[0]['hostId'] == volume1_attached_node: break time.sleep(RETRY_INTERVAL) assert len(volume1.replicas) == 1 assert volume1.replicas[0]['hostId'] == volume1_attached_node delete_and_wait_pod(core_api, pod1_name) volume1 = wait_for_volume_detached(client, volume1_name) volume1_replica_node = volume1.replicas[0]['hostId'] volume1_attached_node = None for node in nodes: if node.name != volume1_replica_node: volume1_attached_node = node.name break assert volume1_attached_node is not None pod1['spec']['nodeSelector'] = \ {"kubernetes.io/hostname": volume1_attached_node} create_and_wait_pod(core_api, pod1) for _ in range(RETRY_COUNTS): volume1 = client.by_id_volume(volume1_name) assert volume1[VOLUME_FIELD_ROBUSTNESS] == VOLUME_ROBUSTNESS_HEALTHY if len(volume1.replicas) == 1 and \ volume1.replicas[0]['hostId'] == volume1_attached_node: break time.sleep(RETRY_INTERVAL) assert len(volume1.replicas) == 1 assert volume1.replicas[0]['hostId'] == volume1_attached_node delete_and_wait_pod(core_api, pod1_name) wait_for_volume_detached(client, volume1_name) # Case 2: Test that Longhorn prioritizes deleting replicas on the same node node1 = nodes[0] node2 = nodes[1] node3 = nodes[2] client.update(node1, allowScheduling=True, tags=["AVAIL"]) client.update(node2, allowScheduling=True, tags=["AVAIL"]) replica_node_soft_anti_affinity_setting = \ client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY) try: client.update(replica_node_soft_anti_affinity_setting, value="true") except Exception as e: print( "Exception when update " "Replica Node Level Soft Anti-Affinity setting", replica_node_soft_anti_affinity_setting, e) volume2_name = volume_name + "-2" volume2_size = str(500 * Mi) pv2_name = volume2_name + "-pv" pvc2_name = volume2_name + "-pvc" pod2_name = volume2_name + "-pod" pod2 = pod pod2['metadata']['name'] = pod2_name volume2 = client.create_volume(name=volume2_name, size=volume2_size, numberOfReplicas=3, nodeSelector=["AVAIL"], dataLocality="best-effort") volume2 = wait_for_volume_detached(client, volume2_name) volume2 = client.by_id_volume(volume2_name) create_pv_for_volume(client, core_api, volume2, pv2_name) create_pvc_for_volume(client, core_api, volume2, pvc2_name) volume2 = client.by_id_volume(volume2_name) pod2['spec']['volumes'] = [{ "name": "pod-data", "persistentVolumeClaim": { "claimName": pvc2_name } }] pod2['spec']['nodeSelector'] = {"kubernetes.io/hostname": node3.name} create_and_wait_pod(core_api, pod2) volume2 = wait_for_volume_healthy(client, volume2_name) for replica in volume2.replicas: assert replica["hostId"] != node3.name volume2.updateReplicaCount(replicaCount=2) # 2 Healthy replicas and 1 replica failed to schedule # The failed to schedule replica is the local replica on node3 volume2 = wait_for_volume_replica_count(client, volume2_name, 3) volume2 = client.by_id_volume(volume2_name) volume2_healthy_replicas = [] for replica in volume2.replicas: if replica.running is True: volume2_healthy_replicas.append(replica) assert len(volume2_healthy_replicas) == 2 volume2_rep1 = volume2_healthy_replicas[0] volume2_rep2 = volume2_healthy_replicas[1] assert volume2_rep1["hostId"] != volume2_rep2["hostId"] delete_and_wait_pod(core_api, pod2_name) wait_for_volume_detached(client, volume2_name) # Case 3: Test that the volume is not corrupted if there is an unexpected # detachment during building local replica client.update(node1, allowScheduling=True, tags=[]) client.update(node2, allowScheduling=True, tags=[]) replica_node_soft_anti_affinity_setting = \ client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY) try: client.update(replica_node_soft_anti_affinity_setting, value="false") except Exception as e: print( "Exception when update " "Replica Node Level Soft Anti-Affinity setting", replica_node_soft_anti_affinity_setting, e) volume3_name = volume_name + "-3" volume3_size = str(1 * Gi) volume3_data_path = "/data/test" pv3_name = volume3_name + "-pv" pvc3_name = volume3_name + "-pvc" pod3_name = volume3_name + "-pod" pod3 = pod pod3['metadata']['name'] = pod3_name volume3 = client.create_volume(name=volume3_name, size=volume3_size, numberOfReplicas=1) volume3 = wait_for_volume_detached(client, volume3_name) volume3 = client.by_id_volume(volume3_name) create_pv_for_volume(client, core_api, volume3, pv3_name) create_pvc_for_volume(client, core_api, volume3, pvc3_name) volume3 = client.by_id_volume(volume3_name) pod3['spec']['volumes'] = [{ "name": "pod-data", "persistentVolumeClaim": { "claimName": pvc3_name } }] pod3['spec']['nodeSelector'] = {"kubernetes.io/hostname": node3.name} create_and_wait_pod(core_api, pod3) volume3 = wait_for_volume_healthy(client, volume3_name) write_pod_volume_random_data(core_api, pod3_name, volume3_data_path, DATA_SIZE_IN_MB_4) volume3.updateDataLocality(dataLocality="best-effort") volume3 = client.by_id_volume(volume3_name) if volume3.replicas[0]['hostId'] != node3.name: wait_for_rebuild_start(client, volume3_name) volume3 = client.by_id_volume(volume3_name) assert len(volume3.replicas) == 2 wait_for_rebuild_complete(client, volume3_name) volume3 = wait_for_volume_replica_count(client, volume3_name, 1) assert volume3.replicas[0]["hostId"] == node3.name delete_and_wait_pod(core_api, pod3_name) pod3['spec']['nodeSelector'] = {"kubernetes.io/hostname": node1.name} create_and_wait_pod(core_api, pod3) wait_for_rebuild_start(client, volume3_name) crash_engine_process_with_sigkill(client, core_api, volume3_name) delete_and_wait_pod(core_api, pod3_name) wait_for_volume_detached(client, volume3_name) volume3 = client.by_id_volume(volume3_name) assert len(volume3.replicas) == 1 assert volume3.replicas[0]["hostId"] == node3.name create_and_wait_pod(core_api, pod3) wait_for_rebuild_start(client, volume3_name) volume3 = client.by_id_volume(volume3_name) assert len(volume3.replicas) == 2 wait_for_rebuild_complete(client, volume3_name) # Wait for deletion of extra replica volume3 = wait_for_volume_replica_count(client, volume3_name, 1) assert volume3.replicas[0]["hostId"] == node1.name assert volume3.replicas[0]["mode"] == "RW" assert volume3.replicas[0]["running"] is True delete_and_wait_pod(core_api, pod3_name) wait_for_volume_detached(client, volume3_name) # Case 4: Make sure failed to schedule local replica doesn't block the # the creation of other replicas. replica_node_soft_anti_affinity_setting = \ client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY) try: client.update(replica_node_soft_anti_affinity_setting, value="false") except Exception as e: print( "Exception when update " "Replica Node Level Soft Anti-Affinity setting", replica_node_soft_anti_affinity_setting, e) client.update(node3, allowScheduling=False) volume4_name = volume_name + "-4" volume4_size = str(1 * Gi) volume4 = client.create_volume(name=volume4_name, size=volume4_size, numberOfReplicas=1, dataLocality="best-effort") volume4 = wait_for_volume_detached(client, volume4_name) volume4 = client.by_id_volume(volume4_name) volume4_replica_name = volume4.replicas[0]["name"] volume4.attach(hostId=node3.name) wait_for_volume_healthy(client, volume4_name) volume4 = client.by_id_volume(volume4_name) assert len(volume4.replicas) == 2 for replica in volume4.replicas: if replica["name"] == volume4_replica_name: assert replica["running"] is True assert replica["mode"] == "RW" else: assert replica["running"] is False assert replica["mode"] == "" assert volume4.conditions.scheduled.reason == \ "LocalReplicaSchedulingFailure" volume4 = volume4.updateReplicaCount(replicaCount=3) volume4 = wait_for_volume_degraded(client, volume4_name) v4_node1_replica_count = 0 v4_node2_replica_count = 0 v4_failed_replica_count = 0 for replica in volume4.replicas: if replica["hostId"] == node1.name: v4_node1_replica_count += 1 elif replica["hostId"] == node2.name: v4_node2_replica_count += 1 elif replica["hostId"] == "": v4_failed_replica_count += 1 assert v4_node1_replica_count == 1 assert v4_node2_replica_count == 1 assert v4_failed_replica_count > 0 volume4 = volume4.updateReplicaCount(replicaCount=2) volume4 = wait_for_volume_replica_count(client, volume4_name, 3) v4_node1_replica_count = 0 v4_node2_replica_count = 0 v4_failed_replica_count = 0 for replica in volume4.replicas: if replica["hostId"] == node1.name: v4_node1_replica_count += 1 elif replica["hostId"] == node2.name: v4_node2_replica_count += 1 elif replica["hostId"] == "": v4_failed_replica_count += 1 assert v4_node1_replica_count == 1 assert v4_node2_replica_count == 1 assert v4_failed_replica_count > 0 volume4 = volume4.updateReplicaCount(replicaCount=1) volume4 = wait_for_volume_replica_count(client, volume4_name, 2) v4_node1_replica_count = 0 v4_node2_replica_count = 0 v4_failed_replica_count = 0 for replica in volume4.replicas: if replica["hostId"] == node1.name: v4_node1_replica_count += 1 elif replica["hostId"] == node2.name: v4_node2_replica_count += 1 elif replica["hostId"] == "": v4_failed_replica_count += 1 assert v4_node1_replica_count + v4_node2_replica_count == 1 assert v4_failed_replica_count == 1 volume4 = volume4.updateDataLocality(dataLocality="disabled") volume4 = volume4.updateReplicaCount(replicaCount=2) running_replica_count = 0 for _ in range(RETRY_COUNTS): volume4 = client.by_id_volume(volume4_name) running_replica_count = 0 for r in volume4.replicas: if r.failedAt == "" and r.running is True: running_replica_count += 1 if running_replica_count == 2: break time.sleep(RETRY_INTERVAL) assert running_replica_count == 2 v4_node1_replica_count = 0 v4_node2_replica_count = 0 v4_node3_replica_count = 0 for replica in volume4.replicas: wait_for_replica_running(client, volume4_name, replica["name"]) if replica["hostId"] == node1.name: v4_node1_replica_count += 1 elif replica["hostId"] == node2.name: v4_node2_replica_count += 1 elif replica["hostId"] == node3.name: v4_node3_replica_count += 1 assert v4_node1_replica_count == 1 assert v4_node2_replica_count == 1 assert v4_node3_replica_count == 0