def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): # NOQA break # error when creating volume with duplicate jobs with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1() + create_jobs1()) assert "duplicate job" in str(e.value) client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1()) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def ha_simple_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == size assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["created"] != "" assert volume["baseImage"] == base_image host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) ha_rebuild_replica_test(client, volume_name) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_replica_scheduler_no_disks(client): # NOQA nodes = client.list_node() # delete all disks on each node for node in nodes: disks = node["disks"] name = node["name"] # set allowScheduling to false for fsid, disk in disks.iteritems(): disk["allowScheduling"] = False update_disks = get_update_disks(disks) node = node.diskUpdate(disks=update_disks) for fsid, disk in node["disks"].iteritems(): # wait for node controller update disk status wait_for_disk_status(client, name, fsid, "allowScheduling", False) wait_for_disk_status(client, name, fsid, "storageScheduled", 0) node = client.by_id_node(name) for fsid, disk in node["disks"].iteritems(): assert not disk["allowScheduling"] node = node.diskUpdate(disks=[]) node = common.wait_for_disk_update(client, name, 0) assert len(node["disks"]) == 0 # test there's no disk fit for volume vol_name = common.generate_volume_name() volume = client.create_volume(name=vol_name, size=SIZE, numberOfReplicas=len(nodes)) volume = common.wait_for_volume_condition_scheduled(client, vol_name, "status", CONDITION_STATUS_FALSE) client.delete(volume) common.wait_for_volume_delete(client, vol_name)
def test_replica_scheduler_exceed_over_provisioning(client): # NOQA over_provisioning_setting = client.by_id_setting( SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE) old_provisioning_setting = over_provisioning_setting["value"] # set storage over provisioning percentage to 100 over_provisioning_setting = client.update(over_provisioning_setting, value="100") # test exceed over provisioning limit couldn't be scheduled nodes = client.list_node() for node in nodes: disks = node["disks"] for fsid, disk in disks.iteritems(): disk["storageReserved"] = \ disk["storageMaximum"] - 1*Gi update_disks = get_update_disks(disks) node = node.diskUpdate(disks=update_disks) disks = node["disks"] for fsid, disk in disks.iteritems(): wait_for_disk_status(client, node["name"], fsid, "storageReserved", disk["storageMaximum"] - 1*Gi) vol_name = common.generate_volume_name() volume = client.create_volume(name=vol_name, size=str(2*Gi), numberOfReplicas=len(nodes)) volume = common.wait_for_volume_condition_scheduled(client, vol_name, "status", CONDITION_STATUS_FALSE) client.delete(volume) common.wait_for_volume_delete(client, vol_name) client.update(over_provisioning_setting, value=old_provisioning_setting)
def test_ha_prohibit_deleting_last_replica(client, volume_name): # NOQA volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=1) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 1 assert volume["state"] == "detached" assert volume["created"] != "" host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 1 replica0 = volume["replicas"][0] with pytest.raises(Exception) as e: volume.replicaRemove(name=replica0["name"]) assert "no other healthy replica available" in str(e.value) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def ha_salvage_test(client, volume_name, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["created"] != "" assert volume["baseImage"] == base_image host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 2 replica0_name = volume["replicas"][0]["name"] replica1_name = volume["replicas"][1]["name"] data = write_volume_random_data(volume) common.k8s_delete_replica_pods_for_volume(volume_name) volume = common.wait_for_volume_faulted(client, volume_name) assert len(volume["replicas"]) == 2 assert volume["replicas"][0]["failedAt"] != "" assert volume["replicas"][1]["failedAt"] != "" volume.salvage(names=[replica0_name, replica1_name]) volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == 2 assert volume["replicas"][0]["failedAt"] == "" assert volume["replicas"][1]["failedAt"] == "" volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) check_volume_data(volume, data) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_csi_expansion_with_size_round_up(client, core_api): # NOQA """ test expand longhorn volume 1. Create longhorn volume with size '1Gi' 2. Attach, write data, and detach 3. Expand volume size to '2000000000/2G' and check if size round up '2000683008' 4. Attach, write data, and detach 5. Expand volume size to '2Gi' and check if size is '2147483648' 6. Attach, write data, and detach """ volume_name = generate_volume_name() volume = create_and_check_volume(client, volume_name, 2, str(1 * Gi)) self_hostId = get_self_host_id() volume.attach(hostId=self_hostId, disableFrontend=False) volume = wait_for_volume_healthy(client, volume_name) test_data = write_volume_random_data(volume) volume.detach(hostId="") volume = wait_for_volume_detached(client, volume_name) volume.expand(size="2000000000") wait_for_volume_expansion(client, volume_name) volume = client.by_id_volume(volume_name) assert volume.size == "2000683008" self_hostId = get_self_host_id() volume.attach(hostId=self_hostId, disableFrontend=False) volume = wait_for_volume_healthy(client, volume_name) check_volume_data(volume, test_data, False) test_data = write_volume_random_data(volume) volume.detach(hostId="") volume = wait_for_volume_detached(client, volume_name) volume.expand(size=str(2 * Gi)) wait_for_volume_expansion(client, volume_name) volume = client.by_id_volume(volume_name) assert volume.size == "2147483648" self_hostId = get_self_host_id() volume.attach(hostId=self_hostId, disableFrontend=False) volume = wait_for_volume_healthy(client, volume_name) check_volume_data(volume, test_data, False) volume.detach(hostId="") volume = wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name)
def test_tag_scheduling_failure(client, node_default_tags): # NOQA """ Test that scheduling fails if no Nodes/Disks with the requested Tags are available. """ invalid_tag_cases = [ # Only one Disk Tag exists. { "disk": ["doesnotexist", "ssd"], "node": [] }, # Only one Node Tag exists. { "disk": [], "node": ["doesnotexist", "main"] } ] for tags in invalid_tag_cases: volume_name = generate_volume_name() # NOQA with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3, diskSelector=tags["disk"], nodeSelector=tags["node"]) assert "does not exist" in str(e.value) unsatisfied_tag_cases = [ { "disk": [], "node": ["main", "fallback"] }, { "disk": ["ssd", "m2"], "node": [] } ] for tags in unsatisfied_tag_cases: volume_name = generate_volume_name() client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3, diskSelector=tags["disk"], nodeSelector=tags["node"]) volume = wait_for_volume_detached(client, volume_name) assert volume["diskSelector"] == tags["disk"] assert volume["nodeSelector"] == tags["node"] wait_scheduling_failure(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume["name"]) volumes = client.list_volume() assert len(volumes) == 0
def test_recurring_job(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): # NOQA break volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) jobs = create_jobs1() volume.recurringUpdate(jobs=jobs) volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) job_backup2 = { "name": "backup2", "cron": "* * * * *", "task": "backup", "retain": 2 } volume.recurringUpdate(jobs=[jobs[0], job_backup2]) # 5 minutes time.sleep(300) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot["removed"] is False: count += 1 # 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head assert count == 6 volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_recurring_job_in_volume_creation(set_random_backupstore, client, volume_name): # NOQA """ Test create volume with recurring jobs 1. Create volume with recurring jobs though Longhorn API 2. Verify the recurring jobs run correctly """ host_id = get_self_host_id() # error when creating volume with duplicate jobs with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1() + create_jobs1()) assert "duplicate job" in str(e.value) client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1()) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) # wait until the beginning of an even minute wait_until_begin_of_an_even_minute() # wait until the 10th second of an even minute # to avoid writing data at the same time backup is taking time.sleep(10) write_volume_random_data(volume) time.sleep(150) # 2.5 minutes write_volume_random_data(volume) time.sleep(150) # 2.5 minutes check_jobs1_result(volume) volume = volume.detach(hostId="") common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_recurring_job(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): # NOQA break volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) jobs = create_jobs1() volume.recurringUpdate(jobs=jobs) volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) job_backup2 = {"name": "backup2", "cron": "* * * * *", "task": "backup", "retain": 2} volume.recurringUpdate(jobs=[jobs[0], job_backup2]) # 5 minutes time.sleep(300) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot["removed"] is False: count += 1 # 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head assert count == 6 volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA """ Test create volume with recurring jobs 1. Create volume with recurring jobs though Longhorn API 2. Verify the recurring jobs run correctly """ for host_id, client in iter(clients.items()): # NOQA break set_random_backupstore(client) # error when creating volume with duplicate jobs with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1() + create_jobs1()) assert "duplicate job" in str(e.value) client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1()) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_engine_image_incompatible(client, core_api, volume_name): # NOQA """ Test incompatible engine images 1. Deploy incompatible engine images 2. Make sure their state are `incompatible` once deployed. """ images = client.list_engine_image() assert len(images) == 1 assert images[0].default assert images[0].state == "ready" cli_v = images[0].cliAPIVersion # cli_minv = images[0].cliAPIMinVersion ctl_v = images[0].controllerAPIVersion ctl_minv = images[0].controllerAPIMinVersion data_v = images[0].dataFormatVersion data_minv = images[0].dataFormatMinVersion fail_cli_v_image = common.get_compatibility_test_image( cli_v - 1, cli_v - 1, ctl_v, ctl_minv, data_v, data_minv) img = client.create_engine_image(image=fail_cli_v_image) img_name = img.name img = wait_for_engine_image_state(client, img_name, "incompatible") assert img.state == "incompatible" assert img.cliAPIVersion == cli_v - 1 assert img.cliAPIMinVersion == cli_v - 1 client.delete(img) wait_for_engine_image_deletion(client, core_api, img.name) fail_cli_minv_image = common.get_compatibility_test_image( cli_v + 1, cli_v + 1, ctl_v, ctl_minv, data_v, data_minv) img = client.create_engine_image(image=fail_cli_minv_image) img_name = img.name img = wait_for_engine_image_state(client, img_name, "incompatible") assert img.state == "incompatible" assert img.cliAPIVersion == cli_v + 1 assert img.cliAPIMinVersion == cli_v + 1 client.delete(img) wait_for_engine_image_deletion(client, core_api, img.name)
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, vol_name, backing_image, test_data): # NOQA write_pod_volume_data(core_api, pod_name, test_data) volume = client.by_id_volume(vol_name) snap = create_snapshot(client, vol_name) volume.snapshotBackup(name=snap.name) common.wait_for_backup_completion(client, vol_name, snap.name) bv, b = common.find_backup(client, vol_name, snap.name) pod2_name = 'csi-backup-test-2' vol2_name = create_and_wait_csi_pod( pod2_name, client, core_api, csi_pv, pvc, pod_make, backing_image, b.url) volume2 = client.by_id_volume(vol2_name) resp = read_volume_data(core_api, pod2_name) assert resp == test_data delete_backup(client, bv.name, b.name) delete_and_wait_pod(core_api, pod2_name) client.delete(volume2)
def test_delete_with_provisioned_pv(client, core_api, storage_class, pvc): # NOQA """ Test that deleting a Volume with dynamically provisioned Persistent Volume and Persistent Volume Claim resources successfully deletes the Volume and cleans up those resources. 1. Create a Storage Class to test with. 2. Create a Persistent Volume Claim that requests a Volume from that Storage Class. 3. Wait for the Volume to be provisioned and for the Kubernetes Status to be updated correctly. 4. Attempt to delete the Volume. 5. Verify that the Volume and its associated resources have been deleted. """ pv = provision_and_wait_pv(client, core_api, storage_class, pvc) pv_name = pv.metadata.name volume_name = pv.spec.csi.volume_handle # NOQA volume = client.by_id_volume(volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) wait_delete_pv(core_api, pv_name) wait_delete_pvc(core_api, pvc['metadata']['name'])
def test_engine_image_incompatible(client, volume_name): # NOQA images = client.list_engine_image() assert len(images) == 1 assert images[0]["default"] assert images[0]["state"] == "ready" cli_v = images[0]["cliAPIVersion"] # cli_minv = images[0]["cliAPIMinVersion"] ctl_v = images[0]["controllerAPIVersion"] ctl_minv = images[0]["controllerAPIMinVersion"] data_v = images[0]["dataFormatVersion"] data_minv = images[0]["dataFormatMinVersion"] fail_cli_v_image = common.get_compatibility_test_image( cli_v - 1, cli_v - 1, ctl_v, ctl_minv, data_v, data_minv) img = client.create_engine_image(image=fail_cli_v_image) img_name = img["name"] img = wait_for_engine_image_state(client, img_name, "incompatible") assert img["state"] == "incompatible" assert img["cliAPIVersion"] == cli_v - 1 assert img["cliAPIMinVersion"] == cli_v - 1 client.delete(img) fail_cli_minv_image = common.get_compatibility_test_image( cli_v + 1, cli_v + 1, ctl_v, ctl_minv, data_v, data_minv) img = client.create_engine_image(image=fail_cli_minv_image) img_name = img["name"] img = wait_for_engine_image_state(client, img_name, "incompatible") assert img["state"] == "incompatible" assert img["cliAPIVersion"] == cli_v + 1 assert img["cliAPIMinVersion"] == cli_v + 1 client.delete(img)
def backing_image_basic_operation_test(client, volume_name, bi_name, bi_url): # NOQA """ Test Backing Image APIs. 1. Create a backing image. 2. Create and attach a Volume with the backing image set. 3. Verify that the all disk states in the backing image are "downloaded". 4. Try to use the API to manually clean up one disk for the backing image but get failed. 5. Try to use the API to directly delete the backing image but get failed. 6. Delete the volume. 7. Use the API to manually clean up one disk for the backing image 8. Delete the backing image. """ volume = create_and_check_volume(client, volume_name, 3, str(BACKING_IMAGE_EXT4_SIZE), bi_name) lht_host_id = get_self_host_id() volume.attach(hostId=lht_host_id) volume = wait_for_volume_healthy(client, volume_name) assert volume.backingImage == bi_name assert volume.size == str(BACKING_IMAGE_EXT4_SIZE) random_disk_id = "" backing_image = client.by_id_backing_image(bi_name) assert backing_image.sourceType == BACKING_IMAGE_SOURCE_TYPE_DOWNLOAD assert backing_image.parameters["url"] == bi_url assert backing_image.currentChecksum != "" assert not backing_image.deletionTimestamp assert len(backing_image.diskFileStatusMap) == 3 for disk_id, status in iter(backing_image.diskFileStatusMap.items()): assert status.state == "ready" random_disk_id = disk_id assert random_disk_id != '' with pytest.raises(Exception): backing_image.backingImageCleanup(disks=[random_disk_id]) with pytest.raises(Exception): client.delete(backing_image) client.delete(volume) wait_for_volume_delete(client, volume_name) backing_image = client.by_id_backing_image(bi_name) backing_image.backingImageCleanup(disks=[random_disk_id]) backing_image = wait_for_backing_image_disk_cleanup( client, bi_name, random_disk_id) client.delete(backing_image)
def test_setting_backing_image_auto_cleanup(client, core_api, volume_name): # NOQA """ Test that the Backing Image Cleanup Wait Interval setting works correctly. The default value of setting `BackingImageCleanupWaitInterval` is 60. 1. Create a backing image. 2. Create multiple volumes using the backing image. 3. Attach all volumes, Then: 1. Wait for all volumes can become running. 2. Verify the correct in all volumes. 3. Verify the backing image disk status map. 4. Verify the only backing image file in each disk is reused by multiple replicas. The backing image file path is `<Data path>/<The backing image name>/backing` 4. Unschedule test node to guarantee when replica removed from test node, no new replica can be rebuilt on the test node. 5. Remove all replicas in one disk. Wait for 50 seconds. Then verify nothing changes in the backing image disk state map (before the cleanup wait interval is passed). 6. Modify `BackingImageCleanupWaitInterval` to a small value. Then verify: 1. The download state of the disk containing no replica becomes terminating first, and the entry will be removed from the map later. 2. The related backing image file is removed. 3. The download state of other disks keep unchanged. All volumes still work fine. 7. Delete all volumes. Verify that there will only remain 1 entry in the backing image disk map 8. Delete the backing image. """ # Step 1 create_backing_image_with_matching_url(client, BACKING_IMAGE_NAME, BACKING_IMAGE_QCOW2_URL) # Step 2 volume_names = [volume_name + "-1", volume_name + "-2", volume_name + "-3"] for volume_name in volume_names: volume = create_and_check_volume(client, volume_name, 3, str(BACKING_IMAGE_EXT4_SIZE), BACKING_IMAGE_NAME) # Step 3 lht_host_id = get_self_host_id() for volume_name in volume_names: volume = client.by_id_volume(volume_name) volume.attach(hostId=lht_host_id) wait_for_volume_healthy(client, volume_name) assert volume.backingImage == BACKING_IMAGE_NAME backing_image = client.by_id_backing_image(BACKING_IMAGE_NAME) assert len(backing_image.diskFileStatusMap) == 3 for disk_id, status in iter(backing_image.diskFileStatusMap.items()): assert status.state == "ready" backing_images_in_disk = os.listdir("/var/lib/longhorn/backing-images") assert len(backing_images_in_disk) == 1 assert os.path.exists("/var/lib/longhorn/backing-images/{}/backing".format( backing_images_in_disk[0])) assert os.path.exists( "/var/lib/longhorn/backing-images/{}/backing.cfg".format( backing_images_in_disk[0])) # Step 4 current_host = client.by_id_node(id=lht_host_id) client.update(current_host, allowScheduling=False) wait_for_node_update(client, lht_host_id, "allowScheduling", False) # Step 5 for volume_name in volume_names: volume = client.by_id_volume(volume_name) for replica in volume.replicas: if replica.hostId == lht_host_id: replica_name = replica.name volume.replicaRemove(name=replica_name) # This wait interval should be smaller than the setting value. # Otherwise, the backing image files may be cleaned up. time.sleep(int(BACKING_IMAGE_CLEANUP_WAIT_INTERVAL)) check_backing_image_disk_map_status(client, BACKING_IMAGE_NAME, 3, "ready") # Step 6 update_setting(client, "backing-image-cleanup-wait-interval", "1") check_backing_image_disk_map_status(client, BACKING_IMAGE_NAME, 2, "ready") backing_images_in_disk = os.listdir("/var/lib/longhorn/backing-images") assert len(backing_images_in_disk) == 0 # Step 7 for volume_name in volume_names: volume = client.by_id_volume(volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) check_backing_image_disk_map_status(client, BACKING_IMAGE_NAME, 1, "ready")
def test_ha_simple_recovery(client, volume_name): # NOQA # get a random client volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["created"] != "" host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) volume = client.by_id_volume(volume_name) assert volume["endpoint"] == DEV_PATH + volume_name assert len(volume["replicas"]) == 2 replica0 = volume["replicas"][0] assert replica0["name"] != "" replica1 = volume["replicas"][1] assert replica1["name"] != "" data = write_random_data(volume["endpoint"]) volume = volume.replicaRemove(name=replica0["name"]) # wait until we saw a replica starts rebuilding new_replica_found = False for i in range(RETRY_COUNTS): v = client.by_id_volume(volume_name) for r in v["replicas"]: if r["name"] != replica0["name"] and \ r["name"] != replica1["name"]: new_replica_found = True break if new_replica_found: break time.sleep(RETRY_ITERVAL) assert new_replica_found volume = common.wait_for_volume_healthy(client, volume_name) volume = client.by_id_volume(volume_name) assert volume["state"] == common.VOLUME_STATE_ATTACHED assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY assert len(volume["replicas"]) >= 2 found = False for replica in volume["replicas"]: if replica["name"] == replica1["name"]: found = True break assert found check_data(volume['endpoint'], data) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def cleanup_volume(client, vol_name): # NOQA volume = client.by_id_volume(vol_name) volume.detach() client.delete(volume) common.wait_for_volume_delete(client, vol_name)
def post(self, request): """ Manage the POST request, the Master Node will receive a block and it will check wether it is accepted or not and will add it to the rest of the blockchain accordingly only if the one requesting it is a Relay Node (see user and password above). If the block is rejected because of bad transactions, those transactions are returned to the relay node that made the request. If the block is accepted, the new block is sent to all relay nodes. """ try: # request contains the block, and the address of the miner logger.debug("Block received from %s" % request.data['miner_address']) block_data = request.data['block'] block = Block.parse(block_data) except KeyError: logger.debug("No block given.") return Response({"errors": "No block given."}, status=status.HTTP_406_NOT_ACCEPTABLE) except ParseException as e: logger.debug("Parsing block error.") return Response({"errors": "%s" % e}, status=status.HTTP_406_NOT_ACCEPTABLE) (hash_verify, bad_transactions) = self.server.update_blockchain(block) if hash_verify and len(bad_transactions) == 0: # block is valid logger.debug("Block '%s' successfully added" % block.header) data = {'transactions': []} for transaction in block.transactions: data['transactions'].append(Transaction.serialize(transaction)) for relay_ip in settings.RELAY_IP: logger.debug("Sending block '%s' to relay %s" % (block.header, relay_ip)) client.post(relay_ip, 'blockchain', block_data) client.delete(relay_ip, 'transactions', data) if self.server.balance >= settings.REWARD: self.server.balance -= settings.REWARD miner_transaction = self.server.wallet.create_transaction( request.data['miner_address'], settings.REWARD) client.post(relay_ip, 'transactions', Transaction.serialize(miner_transaction)) response = Response({"detail": "Block successfully added!"}, status=status.HTTP_201_CREATED) else: logger.debug("Block '%s' can NOT be added (bad header or bad TXs)." % block.header) data = {'transactions': []} if len(bad_transactions) > 0: for transaction in bad_transactions: logger.debug("Bad TX '%s'" % transaction.hash) data['transactions'].append(Transaction.serialize(transaction)) # Send to all relays bad TXs, since the miner can request # transactions from any relay for relay_ip in settings.RELAY_IP: logger.debug("Sending bad TXs to relay %s" % relay_ip) client.delete(relay_ip, 'transactions', data) response = Response({"errors": "Bad transactions where found in new block.", "data": data}, status=status.HTTP_406_NOT_ACCEPTABLE) else: response = Response({"errors": "Bad header.", "data": block_data}, status=status.HTTP_406_NOT_ACCEPTABLE) return response
def test_replica_zone_anti_affinity(client, core_api, volume_name, k8s_node_zone_tags): # NOQA """ Test replica scheduler with zone anti-affinity 1. Set zone anti-affinity to hard. 2. Label nodes 1 & 2 with same zone label "zone1". Label node 3 with zone label "zone2". 3. Create a volume with 3 replicas. 4. Wait for volume condition `scheduled` to be false. 5. Label node 2 with zone label "zone3". 6. Wait for volume condition `scheduled` to be success. 7. Clear the volume. 8. Set zone anti-affinity to soft. 9. Change the zone labels on node 1 & 2 & 3 to "zone1". 10. Create a volume. 11. Wait for volume condition `scheduled` to be success. 12. Clean up the replica count, the zone labels and the volume. """ wait_longhorn_node_zone_updated(client) replica_node_soft_anti_affinity_setting = \ client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY) client.update(replica_node_soft_anti_affinity_setting, value="false") replica_zone_soft_anti_affinity_setting = \ client.by_id_setting(SETTING_REPLICA_ZONE_SOFT_ANTI_AFFINITY) client.update(replica_zone_soft_anti_affinity_setting, value="false") volume = create_and_check_volume(client, volume_name) assert volume.conditions.scheduled.status == "False" assert volume.conditions.scheduled.reason == "ReplicaSchedulingFailure" lh_nodes = client.list_node() count = 0 for node in lh_nodes: count += 1 set_k8s_node_zone_label(core_api, node.name, "lh-zone" + str(count)) wait_longhorn_node_zone_updated(client) wait_for_volume_condition_scheduled(client, volume_name, "status", CONDITION_STATUS_TRUE) replica_zone_soft_anti_affinity_setting = \ client.by_id_setting(SETTING_REPLICA_ZONE_SOFT_ANTI_AFFINITY) client.update(replica_zone_soft_anti_affinity_setting, value="true") volume = client.by_id_volume(volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) for node in lh_nodes: set_k8s_node_zone_label(core_api, node.name, "lh-zone1") wait_longhorn_node_zone_updated(client) volume = create_and_check_volume(client, volume_name) wait_for_volume_condition_scheduled(client, volume_name, "status", CONDITION_STATUS_TRUE)
def engine_live_upgrade_rollback_test(client, volume_name, base_image=""): # NOQA default_img = common.get_default_engine_image(client) default_img_name = default_img["name"] default_img = wait_for_engine_image_ref_count(client, default_img_name, 0) cli_v = default_img["cliAPIVersion"] cli_minv = default_img["cliAPIMinVersion"] ctl_v = default_img["controllerAPIVersion"] ctl_minv = default_img["controllerAPIMinVersion"] data_v = default_img["dataFormatVersion"] data_minv = default_img["dataFormatMinVersion"] wrong_engine_upgrade_image = common.get_compatibility_test_image( cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv) new_img = client.create_engine_image(image=wrong_engine_upgrade_image) new_img_name = new_img["name"] new_img = wait_for_engine_image_state(client, new_img_name, "ready") assert new_img["refCount"] == 0 assert new_img["noRefSince"] != "" default_img = common.get_default_engine_image(client) default_img_name = default_img["name"] volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) default_img = wait_for_engine_image_ref_count(client, default_img_name, 1) assert volume["baseImage"] == base_image original_engine_image = volume["engineImage"] assert original_engine_image != wrong_engine_upgrade_image host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) data = write_volume_random_data(volume) volume.engineUpgrade(image=wrong_engine_upgrade_image) volume = client.by_id_volume(volume["name"]) assert volume["engineImage"] == wrong_engine_upgrade_image assert volume["currentImage"] == original_engine_image with pytest.raises(Exception): # this will timeout wait_for_volume_current_image(client, volume_name, wrong_engine_upgrade_image) # rollback volume.engineUpgrade(image=original_engine_image) volume = wait_for_volume_current_image(client, volume_name, original_engine_image) assert volume["engineImage"] == original_engine_image assert volume["currentImage"] == original_engine_image engine = get_volume_engine(volume) assert engine["engineImage"] == original_engine_image assert engine["currentImage"] == original_engine_image volume = common.wait_for_volume_replica_count(client, volume_name, REPLICA_COUNT) check_volume_data(volume, data) assert volume["state"] == common.VOLUME_STATE_ATTACHED assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY # try again, this time let's try detach volume.engineUpgrade(image=wrong_engine_upgrade_image) volume = client.by_id_volume(volume["name"]) assert volume["engineImage"] == wrong_engine_upgrade_image assert volume["currentImage"] == original_engine_image with pytest.raises(Exception): # this will timeout wait_for_volume_current_image(client, volume_name, wrong_engine_upgrade_image) volume = volume.detach() volume = wait_for_volume_current_image(client, volume_name, wrong_engine_upgrade_image) # all the images would be updated assert volume["engineImage"] == wrong_engine_upgrade_image engine = get_volume_engine(volume) assert engine["engineImage"] == wrong_engine_upgrade_image volume = common.wait_for_volume_replica_count(client, volume_name, REPLICA_COUNT) for replica in volume["replicas"]: assert replica["engineImage"] == wrong_engine_upgrade_image # upgrade to the correct image when offline volume.engineUpgrade(image=original_engine_image) volume = client.by_id_volume(volume["name"]) assert volume["engineImage"] == original_engine_image volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert volume["engineImage"] == original_engine_image assert volume["currentImage"] == original_engine_image engine = get_volume_engine(volume) assert engine["engineImage"] == original_engine_image assert engine["currentImage"] == original_engine_image for replica in volume["replicas"]: assert replica["engineImage"] == original_engine_image assert replica["currentImage"] == original_engine_image check_volume_data(volume, data) client.delete(volume) wait_for_volume_delete(client, volume_name) client.delete(new_img)
def test_backup_kubernetes_status(set_random_backupstore, client, core_api, pod): # NOQA """ Test that Backups have KubernetesStatus stored properly when there is an associated PersistentVolumeClaim and Pod. 1. Setup a random backupstore 2. Set settings Longhorn Static StorageClass to `longhorn-static-test` 3. Create a volume and PV/PVC. Verify the StorageClass of PVC 4. Create a Pod using the PVC. 5. Check volume's Kubernetes status to reflect PV/PVC/Pod correctly. 6. Create a backup for the volume. 7. Verify the labels of created backup reflect PV/PVC/Pod status. 8. Restore the backup to a volume. Wait for restoration to complete. 9. Check the volume's Kubernetes Status 1. Make sure the `lastPodRefAt` and `lastPVCRefAt` is snapshot created time 10. Delete the backup and restored volume. 11. Delete PV/PVC/Pod. 12. Verify volume's Kubernetes Status updated to reflect history data. 13. Attach the volume and create another backup. Verify the labels 14. Verify the volume's Kubernetes status. 15. Restore the previous backup to a new volume. Wait for restoration. 16. Verify the restored volume's Kubernetes status. 1. Make sure `lastPodRefAt` and `lastPVCRefAt` matched volume on step 12 """ host_id = get_self_host_id() static_sc_name = "longhorn-static-test" setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC) setting = client.update(setting, value=static_sc_name) assert setting.value == static_sc_name volume_name = "test-backup-kubernetes-status-pod" # NOQA client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = wait_for_volume_detached(client, volume_name) pod_name = "pod-" + volume_name pv_name = "pv-" + volume_name pvc_name = "pvc-" + volume_name create_pv_for_volume(client, core_api, volume, pv_name) create_pvc_for_volume(client, core_api, volume, pvc_name) ret = core_api.list_namespaced_persistent_volume_claim(namespace='default') pvc_found = False for item in ret.items: if item.metadata.name == pvc_name: pvc_found = item break assert pvc_found assert pvc_found.spec.storage_class_name == static_sc_name pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [{ 'name': pod['spec']['containers'][0]['volumeMounts'][0]['name'], 'persistentVolumeClaim': { 'claimName': pvc_name, }, }] create_and_wait_pod(core_api, pod) ks = { 'lastPodRefAt': '', 'lastPVCRefAt': '', 'namespace': 'default', 'pvcName': pvc_name, 'pvName': pv_name, 'pvStatus': 'Bound', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, volume_name, ks) volume = wait_for_volume_healthy(client, volume_name) # Create Backup manually instead of calling create_backup since Kubernetes # is not guaranteed to mount our Volume to the test host. snap = create_snapshot(client, volume_name) volume.snapshotBackup(name=snap.name) wait_for_backup_completion(client, volume_name, snap.name) _, b = find_backup(client, volume_name, snap.name) # Check backup label status = loads(b.labels.get(KUBERNETES_STATUS_LABEL)) assert status == ks # Check backup volume label for _ in range(RETRY_COUNTS): bv = client.by_id_backupVolume(volume_name) if bv is not None and bv.labels is not None: break time.sleep(RETRY_INTERVAL) assert bv is not None and bv.labels is not None status = loads(bv.labels.get(KUBERNETES_STATUS_LABEL)) assert status == ks restore_name = generate_volume_name() client.create_volume(name=restore_name, size=SIZE, numberOfReplicas=2, fromBackup=b.url) wait_for_volume_restoration_completed(client, restore_name) wait_for_volume_detached(client, restore_name) snapshot_created = b.snapshotCreated ks = { 'lastPodRefAt': b.snapshotCreated, 'lastPVCRefAt': b.snapshotCreated, 'namespace': 'default', 'pvcName': pvc_name, # Restoration should not apply PersistentVolume data. 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, restore_name, ks) restore = client.by_id_volume(restore_name) # We need to compare LastPodRefAt and LastPVCRefAt manually since # wait_volume_kubernetes_status only checks for empty or non-empty state. assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"] assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"] delete_backup(client, bv.name, b.name) client.delete(restore) wait_for_volume_delete(client, restore_name) delete_and_wait_pod(core_api, pod_name) delete_and_wait_pvc(core_api, pvc_name) delete_and_wait_pv(core_api, pv_name) # With the Pod, PVC, and PV deleted, the Volume should have both Ref # fields set. Check that a new Backup and Restore will use this instead of # manually populating the Ref fields. ks = { 'lastPodRefAt': 'NOT NULL', 'lastPVCRefAt': 'NOT NULL', 'namespace': 'default', 'pvcName': pvc_name, 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, volume_name, ks) volume = wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) snap = create_snapshot(client, volume_name) volume.snapshotBackup(name=snap.name) volume = wait_for_backup_completion(client, volume_name, snap.name) bv, b = find_backup(client, volume_name, snap.name) new_b = bv.backupGet(name=b.name) status = loads(new_b.labels.get(KUBERNETES_STATUS_LABEL)) # Check each field manually, we have no idea what the LastPodRefAt or the # LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated. assert status['lastPodRefAt'] != snapshot_created assert status['lastPVCRefAt'] != snapshot_created assert status['namespace'] == "default" assert status['pvcName'] == pvc_name assert status['pvName'] == "" assert status['pvStatus'] == "" assert status['workloadsStatus'] == [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] restore_name = generate_volume_name() client.create_volume(name=restore_name, size=SIZE, numberOfReplicas=2, fromBackup=b.url) wait_for_volume_restoration_completed(client, restore_name) wait_for_volume_detached(client, restore_name) ks = { 'lastPodRefAt': status['lastPodRefAt'], 'lastPVCRefAt': status['lastPVCRefAt'], 'namespace': 'default', 'pvcName': pvc_name, 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, restore_name, ks) restore = client.by_id_volume(restore_name) assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"] assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"] # cleanup backupstore_cleanup(client) client.delete(restore) cleanup_volume(client, volume)
def test_recurring_job(clients, volume_name): # NOQA """ Test recurring job 1. Setup a random backupstore 2. Create a volume. 3. Create two jobs 1 job 1: snapshot every one minute, retain 2 1 job 2: backup every two minutes, retain 1 4. Attach the volume. 5. Sleep for 5 minutes 6. Verify we have 4 snapshots total 1. 2 snapshots, 1 backup, 1 volume-head 7. Update jobs to replace the backup job 1. New backup job run every one minute, retain 2 8. Sleep for 5 minutes. 9. We should have 6 snapshots 1. 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head 10. Make sure we have no more than 5 backups. 1. old backup job may have at most 1 backups 2. new backup job may have at most 3 backups 3. FIXME: Seems we should have at most 4 backups? 11. Make sure we have no more than 2 backups in progress 1. FIXME: Seems we should have at most 1 from the new job? """ for host_id, client in iter(clients.items()): # NOQA break set_random_backupstore(client) volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) jobs = create_jobs1() volume.recurringUpdate(jobs=jobs) volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) job_backup2 = { "name": "backup2", "cron": "* * * * *", "task": "backup", "retain": 2 } volume.recurringUpdate(jobs=[jobs[0], job_backup2]) # 5 minutes time.sleep(300) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot.removed is False: count += 1 # 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head assert count == 6 complete_backup_number = 0 in_progress_backup_number = 0 volume = client.by_id_volume(volume_name) for b in volume.backupStatus: assert b.error == "" if b.state == "complete": complete_backup_number += 1 elif b.state == "in_progress": in_progress_backup_number += 1 assert complete_backup_number <= MAX_BACKUP_STATUS_SIZE # 1 from job_backup, 1 from job_backup2 assert in_progress_backup_number <= 2 volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_engine_image(client, core_api, volume_name): # NOQA """ Test Engine Image deployment 1. List Engine Images and validate basic properities. 2. Try deleting default engine image and it should fail. 3. Try creating a duplicate engine image as default and it should fail 4. Get upgrade test image for the same versions 5. Test if the upgrade test image can be deployed and deleted correctly """ # can be leftover default_img = common.get_default_engine_image(client) default_img_name = default_img.name default_img = wait_for_engine_image_ref_count(client, default_img_name, 0) ei_state = get_engine_image_status_value(client, default_img_name) images = client.list_engine_image() assert len(images) == 1 assert images[0].default assert images[0].state == ei_state assert images[0].refCount == 0 assert images[0].gitCommit != "" assert images[0].buildDate != "" cli_v = default_img.cliAPIVersion cli_minv = default_img.cliAPIMinVersion ctl_v = default_img.controllerAPIVersion ctl_minv = default_img.controllerAPIMinVersion data_v = default_img.dataFormatVersion data_minv = default_img.dataFormatMinVersion assert cli_v != 0 assert cli_minv != 0 assert ctl_v != 0 assert ctl_minv != 0 assert data_v != 0 assert data_minv != 0 # delete default image is not allowed with pytest.raises(Exception) as e: client.delete(images[0]) assert "the default engine image" in str(e.value) # duplicate images with pytest.raises(Exception) as e: client.create_engine_image(image=default_img.image) engine_upgrade_image = common.get_upgrade_test_image( cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv) # test if engine image can be created and cleaned up successfully for _ in range(ENGINE_IMAGE_TEST_REPEAT_COUNT): new_img = client.create_engine_image(image=engine_upgrade_image) new_img_name = new_img.name new_img = wait_for_engine_image_state(client, new_img_name, ei_state) assert not new_img.default assert new_img.state == ei_state assert new_img.refCount == 0 assert new_img.cliAPIVersion != 0 assert new_img.cliAPIMinVersion != 0 assert new_img.controllerAPIVersion != 0 assert new_img.controllerAPIMinVersion != 0 assert new_img.dataFormatVersion != 0 assert new_img.dataFormatMinVersion != 0 assert new_img.gitCommit != "" assert new_img.buildDate != "" client.delete(new_img) wait_for_engine_image_deletion(client, core_api, new_img.name)
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" data = write_volume_random_data(volume) snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) _, b = common.find_backup(client, volume_name, snap2["name"]) res_name = common.generate_volume_name() res_volume = client.create_volume(name=res_name, size=size, numberOfReplicas=2, fromBackup=b["url"]) res_volume = common.wait_for_volume_restoration_completed( client, res_name) res_volume = common.wait_for_volume_detached(client, res_name) res_volume = res_volume.attach(hostId=host_id) res_volume = common.wait_for_volume_healthy(client, res_name) check_volume_data(res_volume, data) snapshots = res_volume.snapshotList() # only the backup snapshot + volume-head assert len(snapshots) == 2 backup_snapshot = "" for snap in snapshots: if snap["name"] != "volume-head": backup_snapshot = snap["name"] assert backup_snapshot != "" res_volume.snapshotCreate() snapshots = res_volume.snapshotList() assert len(snapshots) == 3 res_volume.snapshotDelete(name=backup_snapshot) res_volume.snapshotPurge() snapshots = res_volume.snapshotList() assert len(snapshots) == 2 ha_rebuild_replica_test(client, res_name) res_volume = res_volume.detach() res_volume = common.wait_for_volume_detached(client, res_name) client.delete(res_volume) common.wait_for_volume_delete(client, res_name) cleanup_volume(client, volume)
def engine_live_upgrade_rollback_test(client, core_api, volume_name, backing_image=""): # NOQA default_img = common.get_default_engine_image(client) default_img_name = default_img.name default_img = wait_for_engine_image_ref_count(client, default_img_name, 0) cli_v = default_img.cliAPIVersion cli_minv = default_img.cliAPIMinVersion ctl_v = default_img.controllerAPIVersion ctl_minv = default_img.controllerAPIMinVersion data_v = default_img.dataFormatVersion data_minv = default_img.dataFormatMinVersion wrong_engine_upgrade_image = common.get_compatibility_test_image( cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv) new_img = client.create_engine_image(image=wrong_engine_upgrade_image) new_img_name = new_img.name ei_status_value = get_engine_image_status_value(client, new_img_name) new_img = wait_for_engine_image_state(client, new_img_name, ei_status_value) assert new_img.refCount == 0 assert new_img.noRefSince != "" default_img = common.get_default_engine_image(client) default_img_name = default_img.name client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, backingImage=backing_image) volume = common.wait_for_volume_detached(client, volume_name) wait_for_engine_image_ref_count(client, default_img_name, 1) assert volume.backingImage == backing_image original_engine_image = volume.engineImage assert original_engine_image != wrong_engine_upgrade_image host_id = get_self_host_id() volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) data = write_volume_random_data(volume) volume.engineUpgrade(image=wrong_engine_upgrade_image) volume = client.by_id_volume(volume.name) assert volume.engineImage == wrong_engine_upgrade_image assert volume.currentImage == original_engine_image with pytest.raises(Exception): # this will timeout wait_for_volume_current_image(client, volume_name, wrong_engine_upgrade_image) # rollback volume.engineUpgrade(image=original_engine_image) wait_for_volume_current_image(client, volume_name, original_engine_image) volume = wait_for_volume_replicas_mode(client, volume_name, "RW") assert volume.engineImage == original_engine_image assert volume.currentImage == original_engine_image engine = get_volume_engine(volume) assert engine.engineImage == original_engine_image assert engine.currentImage == original_engine_image volume = common.wait_for_volume_replica_count(client, volume_name, REPLICA_COUNT) check_volume_data(volume, data) assert volume.state == common.VOLUME_STATE_ATTACHED assert volume.robustness == common.VOLUME_ROBUSTNESS_HEALTHY # try again, this time let's try detach volume.engineUpgrade(image=wrong_engine_upgrade_image) volume = client.by_id_volume(volume.name) assert volume.engineImage == wrong_engine_upgrade_image assert volume.currentImage == original_engine_image with pytest.raises(Exception): # this will timeout wait_for_volume_current_image(client, volume_name, wrong_engine_upgrade_image) volume.detach(hostId="") volume = wait_for_volume_current_image(client, volume_name, wrong_engine_upgrade_image) # all the images would be updated assert volume.engineImage == wrong_engine_upgrade_image engine = get_volume_engine(volume) assert engine.engineImage == wrong_engine_upgrade_image volume = common.wait_for_volume_replica_count(client, volume_name, REPLICA_COUNT) for replica in volume.replicas: assert replica.engineImage == wrong_engine_upgrade_image # upgrade to the correct image when offline volume.engineUpgrade(image=original_engine_image) volume = wait_for_volume_current_image(client, volume_name, original_engine_image) volume = client.by_id_volume(volume.name) assert volume.engineImage == original_engine_image volume.attach(hostId=host_id) common.wait_for_volume_healthy(client, volume_name) volume = wait_for_volume_replicas_mode(client, volume_name, "RW") assert volume.engineImage == original_engine_image assert volume.currentImage == original_engine_image engine = get_volume_engine(volume) assert engine.engineImage == original_engine_image assert engine.currentImage == original_engine_image check_volume_endpoint(volume) for replica in volume.replicas: assert replica.engineImage == original_engine_image assert replica.currentImage == original_engine_image check_volume_data(volume, data) client.delete(volume) wait_for_volume_delete(client, volume_name) client.delete(new_img) wait_for_engine_image_deletion(client, core_api, new_img.name)
def delete_orphan(client, orphan_name): # NOQA orphan = client.by_id_orphan(orphan_name) client.delete(orphan) wait_for_orphan_delete(client, orphan_name)
def engine_live_upgrade_test(client, core_api, volume_name, backing_image=""): # NOQA default_img = common.get_default_engine_image(client) default_img_name = default_img.name default_img = wait_for_engine_image_ref_count(client, default_img_name, 0) cli_v = default_img.cliAPIVersion cli_minv = default_img.cliAPIMinVersion ctl_v = default_img.controllerAPIVersion ctl_minv = default_img.controllerAPIMinVersion data_v = default_img.dataFormatVersion data_minv = default_img.dataFormatMinVersion engine_upgrade_image = common.get_upgrade_test_image( cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv) new_img = client.create_engine_image(image=engine_upgrade_image) new_img_name = new_img.name ei_status_value = get_engine_image_status_value(client, new_img_name) new_img = wait_for_engine_image_state(client, new_img_name, ei_status_value) assert new_img.refCount == 0 assert new_img.noRefSince != "" default_img = common.get_default_engine_image(client) default_img_name = default_img.name client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, backingImage=backing_image) volume = common.wait_for_volume_detached(client, volume_name) wait_for_engine_image_ref_count(client, default_img_name, 1) assert volume.name == volume_name assert volume.backingImage == backing_image original_engine_image = volume.engineImage assert original_engine_image != engine_upgrade_image host_id = get_self_host_id() volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert volume.engineImage == original_engine_image assert volume.currentImage == original_engine_image engine = get_volume_engine(volume) assert engine.engineImage == original_engine_image assert engine.currentImage == original_engine_image for replica in volume.replicas: assert replica.engineImage == original_engine_image assert replica.currentImage == original_engine_image data = write_volume_random_data(volume) volume.engineUpgrade(image=engine_upgrade_image) wait_for_volume_current_image(client, volume_name, engine_upgrade_image) # Need to wait for Longhorn to get and update the mode for new replicas volume = wait_for_volume_replicas_mode(client, volume_name, "RW") engine = get_volume_engine(volume) assert engine.engineImage == engine_upgrade_image check_volume_endpoint(volume) wait_for_engine_image_ref_count(client, default_img_name, 0) wait_for_engine_image_ref_count(client, new_img_name, 1) count = 0 # old replica may be in deletion process for replica in volume.replicas: if replica.currentImage == engine_upgrade_image: count += 1 assert count == REPLICA_COUNT check_volume_data(volume, data) volume.detach(hostId="") volume = common.wait_for_volume_detached(client, volume_name) assert len(volume.replicas) == REPLICA_COUNT assert volume.engineImage == engine_upgrade_image engine = get_volume_engine(volume) assert engine.engineImage == engine_upgrade_image for replica in volume.replicas: assert replica.engineImage == engine_upgrade_image volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert volume.engineImage == engine_upgrade_image assert volume.currentImage == engine_upgrade_image engine = get_volume_engine(volume) assert engine.engineImage == engine_upgrade_image assert engine.currentImage == engine_upgrade_image check_volume_endpoint(volume) for replica in volume.replicas: assert replica.engineImage == engine_upgrade_image assert replica.currentImage == engine_upgrade_image # Make sure detaching didn't somehow interfere with the data. check_volume_data(volume, data) volume.engineUpgrade(image=original_engine_image) wait_for_volume_current_image(client, volume_name, original_engine_image) volume = wait_for_volume_replicas_mode(client, volume_name, "RW") engine = get_volume_engine(volume) assert engine.engineImage == original_engine_image assert engine.currentImage == original_engine_image check_volume_endpoint(volume) wait_for_engine_image_ref_count(client, default_img_name, 1) new_img = wait_for_engine_image_ref_count(client, new_img_name, 0) assert volume.engineImage == original_engine_image count = 0 # old replica may be in deletion process for replica in volume.replicas: if replica.engineImage == original_engine_image: count += 1 assert count == REPLICA_COUNT check_volume_data(volume, data) volume.detach(hostId="") volume = common.wait_for_volume_detached(client, volume_name) assert len(volume.replicas) == REPLICA_COUNT assert volume.engineImage == original_engine_image engine = get_volume_engine(volume) assert engine.engineImage == original_engine_image for replica in volume.replicas: assert replica.engineImage == original_engine_image client.delete(volume) wait_for_volume_delete(client, volume_name) client.delete(new_img) wait_for_engine_image_deletion(client, core_api, new_img.name)
def test_backup_kubernetes_status(client, core_api, pod): # NOQA """ Test that Backups have KubernetesStatus stored properly when there is an associated PersistentVolumeClaim and Pod. """ host_id = get_self_host_id() static_sc_name = "longhorn-static-test" setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC) setting = client.update(setting, value=static_sc_name) assert setting["value"] == static_sc_name volume_name = "test-backup-kubernetes-status-pod" client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = wait_for_volume_detached(client, volume_name) pod_name = "pod-" + volume_name pv_name = "pv-" + volume_name pvc_name = "pvc-" + volume_name create_pv_for_volume(client, core_api, volume, pv_name) create_pvc_for_volume(client, core_api, volume, pvc_name) ret = core_api.list_namespaced_persistent_volume_claim(namespace='default') pvc_found = False for item in ret.items: if item.metadata.name == pvc_name: pvc_found = item break assert pvc_found assert pvc_found.spec.storage_class_name == static_sc_name pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [{ 'name': pod['spec']['containers'][0]['volumeMounts'][0]['name'], 'persistentVolumeClaim': { 'claimName': pvc_name, }, }] create_and_wait_pod(core_api, pod) ks = { 'lastPodRefAt': '', 'lastPVCRefAt': '', 'namespace': 'default', 'pvcName': pvc_name, 'pvName': pv_name, 'pvStatus': 'Bound', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, volume_name, ks) volume = wait_for_volume_healthy(client, volume_name) # Create Backup manually instead of calling create_backup since Kubernetes # is not guaranteed to mount our Volume to the test host. snap = volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = find_backup(client, volume_name, snap["name"]) new_b = bv.backupGet(name=b["name"]) status = loads(new_b["labels"].get(KUBERNETES_STATUS_LABEL)) assert status == ks restore_name = generate_volume_name() client.create_volume(name=restore_name, size=SIZE, numberOfReplicas=2, fromBackup=b["url"]) wait_for_volume_restoration_completed(client, restore_name) wait_for_volume_detached(client, restore_name) snapshot_created = b["snapshotCreated"] ks = { 'lastPodRefAt': b["snapshotCreated"], 'lastPVCRefAt': b["snapshotCreated"], 'namespace': 'default', 'pvcName': pvc_name, # Restoration should not apply PersistentVolume data. 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, restore_name, ks) restore = client.by_id_volume(restore_name) # We need to compare LastPodRefAt and LastPVCRefAt manually since # wait_volume_kubernetes_status only checks for empty or non-empty state. assert restore["kubernetesStatus"]["lastPodRefAt"] == ks["lastPodRefAt"] assert restore["kubernetesStatus"]["lastPVCRefAt"] == ks["lastPVCRefAt"] bv.backupDelete(name=b["name"]) client.delete(restore) wait_for_volume_delete(client, restore_name) delete_and_wait_pod(core_api, pod_name) delete_and_wait_pvc(core_api, pvc_name) delete_and_wait_pv(core_api, pv_name) # With the Pod, PVC, and PV deleted, the Volume should have both Ref # fields set. Check that a new Backup and Restore will use this instead of # manually populating the Ref fields. ks = { 'lastPodRefAt': 'NOT NULL', 'lastPVCRefAt': 'NOT NULL', 'namespace': 'default', 'pvcName': pvc_name, 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, volume_name, ks) volume = wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) snap = volume.snapshotCreate() volume.snapshotBackup(name=snap["name"]) bv, b = find_backup(client, volume_name, snap["name"]) new_b = bv.backupGet(name=b["name"]) status = loads(new_b["labels"].get(KUBERNETES_STATUS_LABEL)) # Check each field manually, we have no idea what the LastPodRefAt or the # LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated. assert status["lastPodRefAt"] != snapshot_created assert status["lastPVCRefAt"] != snapshot_created assert status["namespace"] == "default" assert status["pvcName"] == pvc_name assert status["pvName"] == "" assert status["pvStatus"] == "" assert status["workloadsStatus"] == [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] restore_name = generate_volume_name() client.create_volume(name=restore_name, size=SIZE, numberOfReplicas=2, fromBackup=b["url"]) wait_for_volume_restoration_completed(client, restore_name) wait_for_volume_detached(client, restore_name) ks = { 'lastPodRefAt': status["lastPodRefAt"], 'lastPVCRefAt': status["lastPVCRefAt"], 'namespace': 'default', 'pvcName': pvc_name, 'pvName': '', 'pvStatus': '', 'workloadsStatus': [{ 'podName': pod_name, 'podStatus': 'Running', 'workloadName': '', 'workloadType': '' }] } wait_volume_kubernetes_status(client, restore_name, ks) restore = client.by_id_volume(restore_name) assert restore["kubernetesStatus"]["lastPodRefAt"] == ks["lastPodRefAt"] assert restore["kubernetesStatus"]["lastPVCRefAt"] == ks["lastPVCRefAt"] bv.backupDelete(name=b["name"]) client.delete(restore) cleanup_volume(client, volume)
def engine_live_upgrade_test(client, volume_name, base_image=""): # NOQA default_img = common.get_default_engine_image(client) default_img_name = default_img["name"] default_img = wait_for_engine_image_ref_count(client, default_img_name, 0) cli_v = default_img["cliAPIVersion"] cli_minv = default_img["cliAPIMinVersion"] ctl_v = default_img["controllerAPIVersion"] ctl_minv = default_img["controllerAPIMinVersion"] data_v = default_img["dataFormatVersion"] data_minv = default_img["dataFormatMinVersion"] engine_upgrade_image = common.get_upgrade_test_image( cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv) new_img = client.create_engine_image(image=engine_upgrade_image) new_img_name = new_img["name"] new_img = wait_for_engine_image_state(client, new_img_name, "ready") assert new_img["refCount"] == 0 assert new_img["noRefSince"] != "" default_img = common.get_default_engine_image(client) default_img_name = default_img["name"] volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) default_img = wait_for_engine_image_ref_count(client, default_img_name, 1) assert volume["name"] == volume_name assert volume["baseImage"] == base_image original_engine_image = volume["engineImage"] assert original_engine_image != engine_upgrade_image host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert volume["engineImage"] == original_engine_image assert volume["currentImage"] == original_engine_image engine = get_volume_engine(volume) assert engine["engineImage"] == original_engine_image assert engine["currentImage"] == original_engine_image for replica in volume["replicas"]: assert replica["engineImage"] == original_engine_image assert replica["currentImage"] == original_engine_image data = write_volume_random_data(volume) volume.engineUpgrade(image=engine_upgrade_image) volume = wait_for_volume_current_image(client, volume_name, engine_upgrade_image) engine = get_volume_engine(volume) assert engine["engineImage"] == engine_upgrade_image default_img = wait_for_engine_image_ref_count(client, default_img_name, 0) new_img = wait_for_engine_image_ref_count(client, new_img_name, 1) count = 0 # old replica may be in deletion process for replica in volume["replicas"]: if replica["currentImage"] == engine_upgrade_image: count += 1 assert count == REPLICA_COUNT check_volume_data(volume, data) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == REPLICA_COUNT assert volume["engineImage"] == engine_upgrade_image engine = get_volume_engine(volume) assert engine["engineImage"] == engine_upgrade_image for replica in volume["replicas"]: assert replica["engineImage"] == engine_upgrade_image volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert volume["engineImage"] == engine_upgrade_image assert volume["currentImage"] == engine_upgrade_image engine = get_volume_engine(volume) assert engine["engineImage"] == engine_upgrade_image assert engine["currentImage"] == engine_upgrade_image for replica in volume["replicas"]: assert replica["engineImage"] == engine_upgrade_image assert replica["currentImage"] == engine_upgrade_image # Make sure detaching didn't somehow interfere with the data. check_volume_data(volume, data) volume.engineUpgrade(image=original_engine_image) volume = wait_for_volume_current_image(client, volume_name, original_engine_image) engine = get_volume_engine(volume) assert engine["engineImage"] == original_engine_image default_img = wait_for_engine_image_ref_count(client, default_img_name, 1) new_img = wait_for_engine_image_ref_count(client, new_img_name, 0) assert volume["engineImage"] == original_engine_image engine = get_volume_engine(volume) assert engine["engineImage"] == original_engine_image count = 0 # old replica may be in deletion process for replica in volume["replicas"]: if replica["engineImage"] == original_engine_image: count += 1 assert count == REPLICA_COUNT check_volume_data(volume, data) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == REPLICA_COUNT assert volume["engineImage"] == original_engine_image engine = get_volume_engine(volume) assert engine["engineImage"] == original_engine_image for replica in volume["replicas"]: assert replica["engineImage"] == original_engine_image client.delete(volume) wait_for_volume_delete(client, volume_name) client.delete(new_img)
def test_recurring_job(set_random_backupstore, client, volume_name): # NOQA """ Test recurring job 1. Setup a random backupstore 2. Create a volume. 3. Create two jobs 1 job 1: snapshot every one minute, retain 2 1 job 2: backup every two minutes, retain 1 4. Attach the volume. Wait until the 10th second since the beginning of an even minute 5. Write some data. Sleep 2.5 minutes. Write some data. Sleep 2.5 minutes 6. Verify we have 4 snapshots total 1. 2 snapshots, 1 backup, 1 volume-head 7. Update jobs to replace the backup job 1. New backup job run every one minute, retain 2 8. Write some data. Sleep 2.5 minutes. Write some data. Sleep 2.5 minutes 9. We should have 6 snapshots 1. 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head 10. Make sure there are exactly 4 completed backups. 1. old backup job completed 2 backups 2. new backup job completed 2 backups 11. Make sure we have no backup in progress """ ''' The timeline looks like this: 0 1 2 3 4 5 6 7 8 9 10 (minute) |W | | W | | |W | | W | | | (write data) | S | S | | S | S | | (job_snap) | | B | B | | | | | | (job_backup1) | | | | | | B | B | | (job_backup2) ''' host_id = get_self_host_id() volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) jobs = create_jobs1() volume.recurringUpdate(jobs=jobs) volume = volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) # wait until the beginning of an even minute wait_until_begin_of_an_even_minute() # wait until the 20th second of an even minute # make sure that snapshot job happens before the backup job time.sleep(20) write_volume_random_data(volume) time.sleep(120) # 2 minutes write_volume_random_data(volume) time.sleep(120) # 2 minutes check_jobs1_result(volume) job_backup2 = { "name": "backup2", "cron": "* * * * *", "task": "backup", "retain": 2 } volume.recurringUpdate(jobs=[jobs[0], job_backup2]) write_volume_random_data(volume) time.sleep(120) # 2 minutes write_volume_random_data(volume) time.sleep(120) # 2 minutes snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot.removed is False: count += 1 # 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head assert count == 6 complete_backup_number = 0 in_progress_backup_number = 0 volume = client.by_id_volume(volume_name) for b in volume.backupStatus: assert b.error == "" if b.state == "complete": complete_backup_number += 1 elif b.state == "in_progress": in_progress_backup_number += 1 # 2 completed backups from job_backup # 2 completed backups from job_backup2 assert complete_backup_number == 4 assert in_progress_backup_number == 0 volume = volume.detach(hostId="") common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def engine_offline_upgrade_test(client, volume_name, base_image=""): # NOQA default_img = common.get_default_engine_image(client) default_img_name = default_img["name"] default_img = wait_for_engine_image_ref_count(client, default_img_name, 0) cli_v = default_img["cliAPIVersion"] cli_minv = default_img["cliAPIMinVersion"] ctl_v = default_img["controllerAPIVersion"] ctl_minv = default_img["controllerAPIMinVersion"] data_v = default_img["dataFormatVersion"] data_minv = default_img["dataFormatMinVersion"] engine_upgrade_image = common.get_upgrade_test_image( cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv) new_img = client.create_engine_image(image=engine_upgrade_image) new_img_name = new_img["name"] new_img = wait_for_engine_image_state(client, new_img_name, "ready") assert new_img["refCount"] == 0 assert new_img["noRefSince"] != "" default_img = common.get_default_engine_image(client) default_img_name = default_img["name"] volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=REPLICA_COUNT, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) default_img = wait_for_engine_image_ref_count(client, default_img_name, 1) original_engine_image = default_img["image"] assert volume["name"] == volume_name assert volume["engineImage"] == original_engine_image assert volume["currentImage"] == original_engine_image assert volume["baseImage"] == base_image # Before our upgrade, write data to the volume first. host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) data = write_volume_random_data(volume) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) volume.engineUpgrade(image=engine_upgrade_image) volume = wait_for_volume_current_image(client, volume_name, engine_upgrade_image) default_img = wait_for_engine_image_ref_count(client, default_img_name, 0) new_img = wait_for_engine_image_ref_count(client, new_img_name, 1) # cannot delete a image in use with pytest.raises(Exception) as e: client.delete(new_img) assert "while being used" in str(e.value) volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) engine = get_volume_engine(volume) assert engine["engineImage"] == engine_upgrade_image assert engine["currentImage"] == engine_upgrade_image for replica in volume["replicas"]: assert replica["engineImage"] == engine_upgrade_image assert replica["currentImage"] == engine_upgrade_image check_volume_data(volume, data) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) volume.engineUpgrade(image=original_engine_image) volume = wait_for_volume_current_image(client, volume_name, original_engine_image) engine = get_volume_engine(volume) assert volume["engineImage"] == original_engine_image assert engine["engineImage"] == original_engine_image for replica in volume["replicas"]: assert replica["engineImage"] == original_engine_image default_img = wait_for_engine_image_ref_count(client, default_img_name, 1) new_img = wait_for_engine_image_ref_count(client, new_img_name, 0) volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) engine = get_volume_engine(volume) assert engine["engineImage"] == original_engine_image assert engine["currentImage"] == original_engine_image for replica in volume["replicas"]: assert replica["engineImage"] == original_engine_image assert replica["currentImage"] == original_engine_image check_volume_data(volume, data) client.delete(volume) wait_for_volume_delete(client, volume_name) client.delete(new_img)
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" data = write_volume_random_data(volume) snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) _, b = common.find_backup(client, volume_name, snap2["name"]) res_name = common.generate_volume_name() res_volume = client.create_volume(name=res_name, size=size, numberOfReplicas=2, fromBackup=b["url"]) res_volume = common.wait_for_volume_detached(client, res_name) res_volume = res_volume.attach(hostId=host_id) res_volume = common.wait_for_volume_healthy(client, res_name) check_volume_data(res_volume, data) snapshots = res_volume.snapshotList() # only the backup snapshot + volume-head assert len(snapshots) == 2 backup_snapshot = "" for snap in snapshots: if snap["name"] != "volume-head": backup_snapshot = snap["name"] assert backup_snapshot != "" res_volume.snapshotCreate() snapshots = res_volume.snapshotList() assert len(snapshots) == 3 res_volume.snapshotDelete(name=backup_snapshot) res_volume.snapshotPurge() snapshots = res_volume.snapshotList() assert len(snapshots) == 2 ha_rebuild_replica_test(client, res_name) res_volume = res_volume.detach() res_volume = common.wait_for_volume_detached(client, res_name) client.delete(res_volume) common.wait_for_volume_delete(client, res_name) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0