def ha_salvage_test(client, volume_name, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["created"] != "" assert volume["baseImage"] == base_image host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 2 replica0_name = volume["replicas"][0]["name"] replica1_name = volume["replicas"][1]["name"] data = write_volume_random_data(volume) common.k8s_delete_replica_pods_for_volume(volume_name) volume = common.wait_for_volume_faulted(client, volume_name) assert len(volume["replicas"]) == 2 assert volume["replicas"][0]["failedAt"] != "" assert volume["replicas"][1]["failedAt"] != "" volume.salvage(names=[replica0_name, replica1_name]) volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == 2 assert volume["replicas"][0]["failedAt"] == "" assert volume["replicas"][1]["failedAt"] == "" volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) check_volume_data(volume, data) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def migration_confirm_test(clients, volume_name, base_image=""): # NOQA client = get_random_client(clients) hosts = clients.keys() host1 = hosts[0] host2 = hosts[1] volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=REPLICA_COUNT, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host1) volume = common.wait_for_volume_healthy(client, volume_name) volume = volume.migrationStart(nodeId=host2) attached_nodes = get_volume_attached_nodes(volume) assert host1 in attached_nodes assert volume["migrationNodeID"] == host2 with pytest.raises(Exception) as e: volume.migrationConfirm() assert "migration is not ready" in str(e.value) volume = common.wait_for_volume_migration_ready(client, volume_name) volume = volume.migrationConfirm() volume = common.wait_for_volume_migration_node(client, volume_name, host2) assert volume["migrationNodeID"] == "" volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name)
def ha_simple_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == size assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["created"] != "" assert volume["baseImage"] == base_image host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) ha_rebuild_replica_test(client, volume_name) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def migration_rollback_test(clients, volume_name, base_image=""): # NOQA client = get_random_client(clients) hosts = clients.keys() host1 = hosts[0] host2 = hosts[1] volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=REPLICA_COUNT, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host1) volume = common.wait_for_volume_healthy(client, volume_name) volume = volume.migrationStart(nodeId=host2) attached_nodes = get_volume_attached_nodes(volume) assert host1 in attached_nodes assert volume["migrationNodeID"] == host2 volume = common.wait_for_volume_migration_ready(client, volume_name) volume = volume.migrationRollback() volume = common.wait_for_volume_migration_node(client, volume_name, host1) assert volume["migrationNodeID"] == "" volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name)
def test_ha_prohibit_deleting_last_replica(client, volume_name): # NOQA volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=1) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 1 assert volume["state"] == "detached" assert volume["created"] != "" host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 1 replica0 = volume["replicas"][0] with pytest.raises(Exception) as e: volume.replicaRemove(name=replica0["name"]) assert "no other healthy replica available" in str(e.value) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): # NOQA break # error when creating volume with duplicate jobs with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1() + create_jobs1()) assert "duplicate job" in str(e.value) client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1()) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_replica_scheduler_update_over_provisioning(client): # NOQA nodes = client.list_node() lht_hostId = get_self_host_id() expect_node_disk = {} for node in nodes: disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == DEFAULT_DISK_PATH: expect_disk = disk expect_disk["fsid"] = fsid expect_node_disk[node["name"]] = expect_disk over_provisioning_setting = client.by_id_setting( SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE) old_provisioning_setting = over_provisioning_setting["value"] # set storage over provisioning percentage to 0 # to test all replica couldn't be scheduled over_provisioning_setting = client.update(over_provisioning_setting, value="0") vol_name = common.generate_volume_name() volume = client.create_volume(name=vol_name, size=SIZE, numberOfReplicas=len(nodes)) volume = common.wait_for_volume_condition_scheduled(client, vol_name, "status", CONDITION_STATUS_FALSE) # set storage over provisioning percentage to 100 over_provisioning_setting = client.update(over_provisioning_setting, value="100") # check volume status volume = common.wait_for_volume_condition_scheduled(client, vol_name, "status", CONDITION_STATUS_TRUE) volume = common.wait_for_volume_detached(client, vol_name) assert volume["state"] == "detached" assert volume["created"] != "" volume.attach(hostId=lht_hostId) volume = common.wait_for_volume_healthy(client, vol_name) node_hosts = [] for node in nodes: node_hosts.append(node["name"]) # check all replica should be scheduled to default disk for replica in volume["replicas"]: id = replica["hostId"] assert id != "" assert replica["running"] expect_disk = expect_node_disk[id] assert replica["diskID"] == expect_disk["fsid"] assert expect_disk["path"] in replica["dataPath"] node_hosts = filter(lambda x: x != id, node_hosts) assert len(node_hosts) == 0 # clean volume and disk cleanup_volume(client, vol_name) client.update(over_provisioning_setting, value=old_provisioning_setting)
def test_hard_anti_affinity_detach(client, volume_name): # NOQA """ Test that volumes with Hard Anti-Affinity are still able to detach and reattach to a node properly, even in degraded state. 1. Create a volume and attach to the current node 2. Generate and write `data` to the volume. 3. Set `soft anti-affinity` to false 4. Disable current node's scheduling. 5. Remove the replica on the current node 1. Verify volume will be in degraded state. 2. Verify volume reports condition `scheduled == false` 6. Detach the volume. 7. Verify that volume only have 2 replicas 1. Unhealthy replica will be removed upon detach. 8. Attach the volume again. 1. Verify volume will be in degraded state. 2. Verify volume reports condition `scheduled == false` 3. Verify only two of three replicas of volume are healthy. 4. Verify the remaining replica doesn't have `replica.HostID`, meaning it's unscheduled 9. Check volume `data` """ volume = create_and_check_volume(client, volume_name) host_id = get_self_host_id() volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) assert len(volume.replicas) == 3 data = write_volume_random_data(volume) setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY) client.update(setting, value="false") node = client.by_id_node(host_id) client.update(node, allowScheduling=False) host_replica = get_host_replica(volume, host_id) volume.replicaRemove(name=host_replica.name) volume = wait_for_volume_degraded(client, volume_name) wait_scheduling_failure(client, volume_name) volume.detach(hostId="") volume = wait_for_volume_detached(client, volume_name) assert len(volume.replicas) == 2 volume.attach(hostId=host_id) # Make sure we're still not getting another successful replica. volume = wait_for_volume_degraded(client, volume_name) wait_scheduling_failure(client, volume_name) assert sum([ 1 for replica in volume.replicas if replica.running and replica.mode == "RW" ]) == 2 assert sum([1 for replica in volume.replicas if not replica.hostId]) == 1 assert len(volume.replicas) == 3 check_volume_data(volume, data) cleanup_volume(client, volume)
def test_soft_anti_affinity_detach(client, volume_name): # NOQA """ Test that volumes with Soft Anti-Affinity can detach and reattach to a node properly. 1. Create a volume and attach to the current node. 2. Generate and write `data` to the volume 3. Set `soft anti-affinity` to true 4. Disable current node's scheduling. 5. Remove the replica on the current node 6. Wait for the new replica to be rebuilt 7. Detach the volume. 8. Verify there are 3 replicas 9. Attach the volume again. Verify there are still 3 replicas 10. Verify the `data`. """ volume = create_and_check_volume(client, volume_name) host_id = get_self_host_id() volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) assert len(volume.replicas) == 3 data = write_volume_random_data(volume) setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY) client.update(setting, value="true") node = client.by_id_node(host_id) client.update(node, allowScheduling=False) replica_names = list(map(lambda replica: replica.name, volume.replicas)) host_replica = get_host_replica(volume, host_id) volume.replicaRemove(name=host_replica.name) wait_new_replica_ready(client, volume_name, replica_names) volume = wait_for_volume_healthy(client, volume_name) volume.detach() volume = wait_for_volume_detached(client, volume_name) assert len(volume.replicas) == 3 volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) assert len(volume.replicas) == 3 check_volume_data(volume, data) cleanup_volume(client, volume)
def ha_simple_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = create_and_check_volume(client, volume_name, 2, size, base_image) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) ha_rebuild_replica_test(client, volume_name) cleanup_volume(client, volume)
def test_hard_anti_affinity_scheduling(client, volume_name): # NOQA """ Test that volumes with Hard Anti-Affinity work as expected. With Hard Anti-Affinity, scheduling on nodes with existing replicas should be forbidden, resulting in "Degraded" state. 1. Create a volume and attach to the current node 2. Generate and write `data` to the volume. 3. Set `soft anti-affinity` to false 4. Disable current node's scheduling. 5. Remove the replica on the current node 1. Verify volume will be in degraded state. 2. Verify volume reports condition `scheduled == false` 3. Verify only two of three replicas of volume are healthy. 4. Verify the remaining replica doesn't have `replica.HostID`, meaning it's unscheduled 6. Check volume `data` """ volume = create_and_check_volume(client, volume_name) host_id = get_self_host_id() volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) assert len(volume.replicas) == 3 data = write_volume_random_data(volume) setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY) client.update(setting, value="false") node = client.by_id_node(host_id) client.update(node, allowScheduling=False) host_replica = get_host_replica(volume, host_id) volume.replicaRemove(name=host_replica.name) # Instead of waiting for timeout and lengthening the tests a significant # amount we can make sure the scheduling isn't working by making sure the # volume becomes Degraded and reports a scheduling error. wait_for_volume_degraded(client, volume_name) wait_scheduling_failure(client, volume_name) # While there are three replicas that should exist to meet the Volume's # request, only two of those volumes should actually be Healthy. volume = client.by_id_volume(volume_name) assert sum([ 1 for replica in volume.replicas if replica.running and replica.mode == "RW" ]) == 2 # Confirm that the final volume is an unscheduled volume. assert sum([1 for replica in volume.replicas if not replica.hostId]) == 1 # Three replicas in total should still exist. assert len(volume.replicas) == 3 check_volume_data(volume, data) cleanup_volume(client, volume)
def volume_iscsi_basic_test(clients, volume_name, base_image=""): # NOQA # get a random client for host_id, client in clients.iteritems(): break volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3, frontend="iscsi", baseImage=base_image) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 3 assert volume["frontend"] == "iscsi" assert volume["baseImage"] == base_image volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == 3 assert volume["state"] == "detached" assert volume["created"] != "" volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == volume["name"] assert volumes[0]["size"] == volume["size"] assert volumes[0]["numberOfReplicas"] == volume["numberOfReplicas"] assert volumes[0]["state"] == volume["state"] assert volumes[0]["created"] == volume["created"] assert volumes[0]["frontend"] == "iscsi" endpoint = get_volume_endpoint(volumes[0]) assert endpoint.startswith("iscsi://") try: dev = iscsi_login(endpoint) volume_rw_test(dev) finally: iscsi_logout(endpoint) volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def revert_random_snapshot(client, core_api, volume_name, pod_manifest, snapshots_md5sum): # NOQA volume = client.by_id_volume(volume_name) host_id = get_self_host_id() pod_name = pod_manifest["metadata"]["name"] # wait for volume healthy if rebuilding deleted replica if len(volume.robustness) != VOLUME_ROBUSTNESS_HEALTHY: wait_for_volume_healthy(client, volume_name) snapshot = get_random_snapshot(snapshots_md5sum) if snapshot is None: print("skipped, no snapshot found", end=" ") return delete_and_wait_pod(core_api, pod_name) wait_for_volume_detached(client, volume_name) volume = client.by_id_volume(volume_name) volume.attach(hostId=host_id, disableFrontend=True) volume = wait_for_volume_healthy_no_frontend(client, volume_name) volume.snapshotRevert(name=snapshot) volume = client.by_id_volume(volume_name) volume.detach() wait_for_volume_detached(client, volume_name) create_and_wait_pod(core_api, pod_manifest) current_md5sum = read_data_md5sum(core_api, pod_name) assert current_md5sum == snapshots_md5sum[snapshot].data_md5sum
def check_volume_data(client, volume_name, data): # NOQA """ Attaches the volume to the current node then compares the volumes data against the passed data. """ volume = client.by_id_volume(volume_name) volume.attach(hostId=common.get_self_host_id()) volume = common.wait_for_volume_healthy(client, volume_name) common.check_volume_data(volume, data) volume.detach(hostId="") volume = common.wait_for_volume_detached(client, volume_name)
def backupstore_test(client, host_id, volname, size): volume = client.by_id_volume(volname) volume.snapshotCreate() data = write_volume_random_data(volume) snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) bv, b = common.find_backup(client, volname, snap2["name"]) new_b = bv.backupGet(name=b["name"]) assert new_b["name"] == b["name"] assert new_b["url"] == b["url"] assert new_b["snapshotName"] == b["snapshotName"] assert new_b["snapshotCreated"] == b["snapshotCreated"] assert new_b["created"] == b["created"] assert new_b["volumeName"] == b["volumeName"] assert new_b["volumeSize"] == b["volumeSize"] assert new_b["volumeCreated"] == b["volumeCreated"] # test restore restoreName = generate_volume_name() volume = client.create_volume(name=restoreName, size=size, numberOfReplicas=2, fromBackup=b["url"]) volume = common.wait_for_volume_detached(client, restoreName) assert volume["name"] == restoreName assert volume["size"] == size assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, restoreName) check_volume_data(volume, data) volume = volume.detach() volume = common.wait_for_volume_detached(client, restoreName) client.delete(volume) volume = wait_for_volume_delete(client, restoreName) bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap2["name"]: found = True break assert not found
def backing_image_basic_operation_test(client, volume_name, bi_name, bi_url): # NOQA """ Test Backing Image APIs. 1. Create a backing image. 2. Create and attach a Volume with the backing image set. 3. Verify that the all disk states in the backing image are "downloaded". 4. Try to use the API to manually clean up one disk for the backing image but get failed. 5. Try to use the API to directly delete the backing image but get failed. 6. Delete the volume. 7. Use the API to manually clean up one disk for the backing image 8. Delete the backing image. """ volume = create_and_check_volume(client, volume_name, 3, str(BACKING_IMAGE_EXT4_SIZE), bi_name) lht_host_id = get_self_host_id() volume.attach(hostId=lht_host_id) volume = wait_for_volume_healthy(client, volume_name) assert volume.backingImage == bi_name assert volume.size == str(BACKING_IMAGE_EXT4_SIZE) random_disk_id = "" backing_image = client.by_id_backing_image(bi_name) assert backing_image.sourceType == BACKING_IMAGE_SOURCE_TYPE_DOWNLOAD assert backing_image.parameters["url"] == bi_url assert backing_image.currentChecksum != "" assert not backing_image.deletionTimestamp assert len(backing_image.diskFileStatusMap) == 3 for disk_id, status in iter(backing_image.diskFileStatusMap.items()): assert status.state == "ready" random_disk_id = disk_id assert random_disk_id != '' with pytest.raises(Exception): backing_image.backingImageCleanup(disks=[random_disk_id]) with pytest.raises(Exception): client.delete(backing_image) client.delete(volume) wait_for_volume_delete(client, volume_name) backing_image = client.by_id_backing_image(bi_name) backing_image.backingImageCleanup(disks=[random_disk_id]) backing_image = wait_for_backing_image_disk_cleanup( client, bi_name, random_disk_id) client.delete(backing_image)
def backup_test(clients, volume_name, size, base_image=""): # NOQA for host_id, client in clients.iteritems(): break volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == size assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["baseImage"] == base_image lht_hostId = get_self_host_id() volume = volume.attach(hostId=lht_hostId) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" backupstore_test(client, lht_hostId, volume_name, size) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) volume = wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_volume_update_replica_count(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): break replica_count = 3 volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=replica_count) volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == replica_count volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) replica_count = 5 volume = volume.updateReplicaCount(replicaCount=replica_count) volume = common.wait_for_volume_degraded(client, volume_name) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == replica_count old_replica_count = replica_count replica_count = 2 volume = volume.updateReplicaCount(replicaCount=replica_count) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == old_replica_count volume.replicaRemove(name=volume["replicas"][0]["name"]) volume.replicaRemove(name=volume["replicas"][1]["name"]) volume.replicaRemove(name=volume["replicas"][2]["name"]) volume = common.wait_for_volume_replica_count(client, volume_name, replica_count) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == replica_count client.delete(volume) wait_for_volume_delete(client, volume_name)
def test_soft_anti_affinity_scheduling(client, volume_name): # NOQA """ Test that volumes with Soft Anti-Affinity work as expected. With Soft Anti-Affinity, a new replica should still be scheduled on a node with an existing replica, which will result in "Healthy" state but limited redundancy. 1. Create a volume and attach to the current node 2. Generate and write `data` to the volume. 3. Set `soft anti-affinity` to true 4. Disable current node's scheduling. 5. Remove the replica on the current node 6. Wait for the volume to complete rebuild. Volume should have 3 replicas. 7. Verify `data` """ volume = create_and_check_volume(client, volume_name) host_id = get_self_host_id() volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) assert len(volume.replicas) == 3 data = write_volume_random_data(volume) setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY) client.update(setting, value="true") node = client.by_id_node(host_id) client.update(node, allowScheduling=False) replica_names = list(map(lambda replica: replica.name, volume.replicas)) host_replica = get_host_replica(volume, host_id) volume.replicaRemove(name=host_replica.name) wait_new_replica_ready(client, volume_name, replica_names) volume = wait_for_volume_healthy(client, volume_name) assert len(volume.replicas) == 3 check_volume_data(volume, data) cleanup_volume(client, volume)
def recurring_job_labels_test(client, labels, volume_name, size=SIZE, base_image=""): # NOQA host_id = get_self_host_id() client.create_volume(name=volume_name, size=size, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) # Simple Backup Job that runs every 2 minutes, retains 1. jobs = [{ "name": RECURRING_JOB_NAME, "cron": "*/2 * * * *", "task": "backup", "retain": 1, "labels": labels }] volume.recurringUpdate(jobs=jobs) volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot["removed"] is False: count += 1 # 1 from Backup, 1 from Volume Head. assert count == 2 # Verify the Labels on the actual Backup. bv = client.by_id_backupVolume(volume_name) backups = bv.backupList() assert len(backups) == 1 b = bv.backupGet(name=backups[0]["name"]) for key, val in labels.iteritems(): assert b["labels"].get(key) == val assert b["labels"].get(RECURRING_JOB_LABEL) == RECURRING_JOB_NAME if base_image: assert b["labels"].get(BASE_IMAGE_LABEL) == base_image # One extra Label from the BaseImage being set. assert len(b["labels"]) == len(labels) + 2 else: # At least one extra Label from RecurringJob. assert len(b["labels"]) == len(labels) + 1 cleanup_volume(client, volume)
def ha_salvage_test(client, volume_name, base_image=""): # NOQA volume = create_and_check_volume(client, volume_name, 2, base_image=base_image) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 2 replica0_name = volume["replicas"][0]["name"] replica1_name = volume["replicas"][1]["name"] data = write_volume_random_data(volume) common.k8s_delete_replica_pods_for_volume(volume_name) volume = common.wait_for_volume_faulted(client, volume_name) assert len(volume["replicas"]) == 2 assert volume["replicas"][0]["failedAt"] != "" assert volume["replicas"][1]["failedAt"] != "" volume.salvage(names=[replica0_name, replica1_name]) volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == 2 assert volume["replicas"][0]["failedAt"] == "" assert volume["replicas"][1]["failedAt"] == "" volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) check_volume_data(volume, data) cleanup_volume(client, volume)
def test_ha_prohibit_deleting_last_replica(client, volume_name): # NOQA volume = create_and_check_volume(client, volume_name, 1) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 1 replica0 = volume["replicas"][0] with pytest.raises(Exception) as e: volume.replicaRemove(name=replica0["name"]) assert "no other healthy replica available" in str(e.value) cleanup_volume(client, volume)
def test_recurring_job_in_volume_creation(set_random_backupstore, client, volume_name): # NOQA """ Test create volume with recurring jobs 1. Create volume with recurring jobs though Longhorn API 2. Verify the recurring jobs run correctly """ host_id = get_self_host_id() # error when creating volume with duplicate jobs with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1() + create_jobs1()) assert "duplicate job" in str(e.value) client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1()) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) # wait until the beginning of an even minute wait_until_begin_of_an_even_minute() # wait until the 10th second of an even minute # to avoid writing data at the same time backup is taking time.sleep(10) write_volume_random_data(volume) time.sleep(150) # 2.5 minutes write_volume_random_data(volume) time.sleep(150) # 2.5 minutes check_jobs1_result(volume) volume = volume.detach(hostId="") common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_recurring_job(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): # NOQA break volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) jobs = create_jobs1() volume.recurringUpdate(jobs=jobs) volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) job_backup2 = { "name": "backup2", "cron": "* * * * *", "task": "backup", "retain": 2 } volume.recurringUpdate(jobs=[jobs[0], job_backup2]) # 5 minutes time.sleep(300) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot["removed"] is False: count += 1 # 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head assert count == 6 volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def backup_labels_test(clients, random_labels, volume_name, size=SIZE, base_image=""): # NOQA for _, client in clients.iteritems(): break host_id = get_self_host_id() volume = create_and_check_volume(client, volume_name, 2, size, base_image) volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" bv, b, _, _ = create_backup(client, volume_name, labels=random_labels) # If we're running the test with a BaseImage, check that this Label is # set properly. backup = bv.backupGet(name=b["name"]) if base_image: assert backup["labels"].get(common.BASE_IMAGE_LABEL) == base_image # One extra Label from the BaseImage being set. assert len(backup["labels"]) == len(random_labels) + 1 else: assert len(backup["labels"]) == len(random_labels) cleanup_volume(client, volume)
def test_tag_scheduling(client, node_default_tags): # NOQA """ Test that scheduling succeeds if there are available Nodes/Disks with the requested Tags. """ host_id = get_self_host_id() tag_specs = [ # Select all Nodes. { "disk": [], "expected": 3, "node": [] }, # Selector works with AND on Disk Tags. { "disk": ["ssd", "nvme"], "expected": 2, "node": [] }, # Selector works with AND on Node Tags. { "disk": [], "expected": 2, "node": ["main", "storage"] }, # Selector works based on combined Disk AND Node selector. { "disk": ["ssd", "nvme"], "expected": 1, "node": ["storage", "main"] } ] for specs in tag_specs: volume_name = generate_volume_name() # NOQA client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3, diskSelector=specs["disk"], nodeSelector=specs["node"]) volume = wait_for_volume_detached(client, volume_name) assert volume["diskSelector"] == specs["disk"] assert volume["nodeSelector"] == specs["node"] volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 3 check_volume_replicas(volume, specs, node_default_tags) cleanup_volume(client, volume)
def test_volume_scheduling_failure(clients, volume_name): # NOQA ''' Test fail to schedule by disable scheduling for all the nodes Also test cannot attach a scheduling failed volume ''' client = get_random_client(clients) nodes = client.list_node() assert len(nodes) > 0 for node in nodes: node = client.update(node, allowScheduling=False) node = common.wait_for_node_update(client, node["id"], "allowScheduling", False) volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3) volume = common.wait_for_volume_condition_scheduled(client, volume_name, "status", CONDITION_STATUS_FALSE) volume = common.wait_for_volume_detached(client, volume_name) self_node = get_self_host_id() with pytest.raises(Exception) as e: volume.attach(hostId=self_node) assert "not scheduled" in str(e.value) for node in nodes: node = client.update(node, allowScheduling=True) node = common.wait_for_node_update(client, node["id"], "allowScheduling", True) volume = common.wait_for_volume_condition_scheduled(client, volume_name, "status", CONDITION_STATUS_TRUE) volume = common.wait_for_volume_detached(client, volume_name) volume = volume.attach(hostId=self_node) volume = common.wait_for_volume_healthy(client, volume_name) endpoint = get_volume_endpoint(volume) assert endpoint != "" volume_rw_test(endpoint) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name)
def test_tag_scheduling_on_update(client, node_default_tags, volume_name): # NOQA """ Test that Replicas get scheduled if a Node/Disk disks updated with the proper Tags. """ tag_spec = { "disk": ["ssd", "m2"], "expected": 1, "node": ["main", "fallback"] } client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3, diskSelector=tag_spec["disk"], nodeSelector=tag_spec["node"]) volume = wait_for_volume_detached(client, volume_name) assert volume["diskSelector"] == tag_spec["disk"] assert volume["nodeSelector"] == tag_spec["node"] wait_scheduling_failure(client, volume_name) host_id = get_self_host_id() node = client.by_id_node(host_id) update_disks = get_update_disks(node["disks"]) update_disks[0]["tags"] = tag_spec["disk"] node = node.diskUpdate(disks=update_disks) set_node_tags(client, node, tag_spec["node"]) scheduled = False for i in range(RETRY_COUNTS): v = client.by_id_volume(volume_name) if v["conditions"]["scheduled"]["status"] == "True": scheduled = True if scheduled: break sleep(RETRY_INTERVAL) assert scheduled volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) nodes = client.list_node() node_mapping = {node["id"]: { "disk": get_update_disks(node["disks"])[0]["tags"], "node": node["tags"] } for node in nodes} assert len(volume["replicas"]) == 3 check_volume_replicas(volume, tag_spec, node_mapping) cleanup_volume(client, volume)
def test_deleting_backup_volume(clients): # NOQA for host_id, client in clients.iteritems(): break lht_hostId = get_self_host_id() volName = generate_volume_name() volume = create_and_check_volume(client, volName) volume.attach(hostId=lht_hostId) volume = common.wait_for_volume_healthy(client, volName) bv, _, snap1, _ = create_backup(client, volName) _, _, snap2, _ = create_backup(client, volName) bv = client.by_id_backupVolume(volName) client.delete(bv) common.wait_for_backup_volume_delete(client, volName) cleanup_volume(client, volume)
def ha_rebuild_replica_test(client, volname): # NOQA volume = client.by_id_volume(volname) assert get_volume_endpoint(volume) == DEV_PATH + volname assert len(volume["replicas"]) == 2 replica0 = volume["replicas"][0] assert replica0["name"] != "" replica1 = volume["replicas"][1] assert replica1["name"] != "" data = write_volume_random_data(volume) volume = volume.replicaRemove(name=replica0["name"]) # wait until we saw a replica starts rebuilding new_replica_found = False for i in range(RETRY_COUNTS): v = client.by_id_volume(volname) for r in v["replicas"]: if r["name"] != replica0["name"] and \ r["name"] != replica1["name"]: new_replica_found = True break if new_replica_found: break time.sleep(RETRY_INTERVAL) assert new_replica_found volume = common.wait_for_volume_healthy(client, volname) volume = client.by_id_volume(volname) assert volume["state"] == common.VOLUME_STATE_ATTACHED assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY assert len(volume["replicas"]) >= 2 found = False for replica in volume["replicas"]: if replica["name"] == replica1["name"]: found = True break assert found check_volume_data(volume, data)
def ha_rebuild_replica_test(client, volname): # NOQA volume = client.by_id_volume(volname) assert get_volume_endpoint(volume) == DEV_PATH + volname assert len(volume["replicas"]) == 2 replica0 = volume["replicas"][0] assert replica0["name"] != "" replica1 = volume["replicas"][1] assert replica1["name"] != "" data = write_volume_random_data(volume) volume = volume.replicaRemove(name=replica0["name"]) # wait until we saw a replica starts rebuilding new_replica_found = False for i in range(RETRY_COUNTS): v = client.by_id_volume(volname) for r in v["replicas"]: if r["name"] != replica0["name"] and \ r["name"] != replica1["name"]: new_replica_found = True break if new_replica_found: break time.sleep(RETRY_INTERVAL) assert new_replica_found volume = common.wait_for_volume_healthy(client, volname) volume = client.by_id_volume(volname) assert volume["state"] == common.VOLUME_STATE_ATTACHED assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY assert len(volume["replicas"]) >= 2 found = False for replica in volume["replicas"]: if replica["name"] == replica1["name"]: found = True break assert found check_volume_data(volume, data)
def test_recurring_job(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): # NOQA break volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) jobs = create_jobs1() volume.recurringUpdate(jobs=jobs) volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) job_backup2 = {"name": "backup2", "cron": "* * * * *", "task": "backup", "retain": 2} volume.recurringUpdate(jobs=[jobs[0], job_backup2]) # 5 minutes time.sleep(300) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot["removed"] is False: count += 1 # 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head assert count == 6 volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_xfs_pv_existing_volume(client, core_api, pod_manifest): # NOQA """ Test create PV with existing XFS filesystem 1. Create a volume 2. Create PV/PVC for the existing volume, specify `xfs` as filesystem 3. Attach the volume to the current node. 4. Format it to `xfs` 5. Create a POD using the volume FIXME: We should write data in step 4 and validate the data in step 5, make sure the disk won't be reformatted """ volume_name = generate_volume_name() volume = create_and_check_volume(client, volume_name) create_pv_for_volume(client, core_api, volume, volume_name, "xfs") create_pvc_for_volume(client, core_api, volume, volume_name) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) cmd = ['mkfs.xfs', get_volume_endpoint(volume)] subprocess.check_call(cmd) volume = volume.detach() volume = wait_for_volume_detached(client, volume_name) pod_manifest['spec']['volumes'] = [{ "name": "pod-data", "persistentVolumeClaim": { "claimName": volume_name } }] create_and_wait_pod(core_api, pod_manifest)
def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA """ Test create volume with recurring jobs 1. Create volume with recurring jobs though Longhorn API 2. Verify the recurring jobs run correctly """ for host_id, client in iter(clients.items()): # NOQA break set_random_backupstore(client) # error when creating volume with duplicate jobs with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1() + create_jobs1()) assert "duplicate job" in str(e.value) client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1()) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def setup_migration_test(clients, volume_name, backing_image=""): # NOQA """ Creates a new migratable volume then attaches it to the current node to write some test data on it. """ client = get_random_client(clients) volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=REPLICA_COUNT, backingImage=backing_image, accessMode="rwx", migratable=True) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=common.get_self_host_id()) volume = common.wait_for_volume_healthy(client, volume_name) # write test data data = common.write_volume_random_data(volume) common.check_volume_data(volume, data) volume.detach(hostId="") volume = common.wait_for_volume_detached(client, volume_name) return client, volume, data
def test_attach_without_frontend(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): break volume = create_and_check_volume(client, volume_name) lht_hostId = get_self_host_id() volume.attach(hostId=lht_hostId, disableFrontend=False) common.wait_for_volume_healthy(client, volume_name) volume = client.by_id_volume(volume_name) assert volume["disableFrontend"] is False assert volume["frontend"] == "blockdev" snap1_data = write_volume_random_data(volume) snap1 = volume.snapshotCreate() write_volume_random_data(volume) volume.snapshotCreate() volume.detach() volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=lht_hostId, disableFrontend=True) common.wait_for_volume_healthy(client, volume_name) volume = client.by_id_volume(volume_name) engine = get_volume_engine(volume) assert volume["disableFrontend"] is True assert volume["frontend"] == "blockdev" assert engine["endpoint"] == "" volume.snapshotRevert(name=snap1["name"]) volume.detach() volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=lht_hostId, disableFrontend=False) common.wait_for_volume_healthy(client, volume_name) volume = client.by_id_volume(volume_name) assert volume["disableFrontend"] is False assert volume["frontend"] == "blockdev" check_volume_data(volume, snap1_data) client.delete(volume) wait_for_volume_delete(client, volume_name)
def test_hard_anti_affinity_scheduling(client, volume_name): # NOQA """ Test that volumes with Hard Anti-Affinity work as expected. With Hard Anti-Affinity, scheduling on nodes with existing replicas should be forbidden, resulting in "Degraded" state. """ volume = create_and_check_volume(client, volume_name) host_id = get_self_host_id() volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 3 data = write_volume_random_data(volume) setting = client.by_id_setting(SETTING_REPLICA_SOFT_ANTI_AFFINITY) client.update(setting, value="false") node = client.by_id_node(host_id) client.update(node, allowScheduling=False) host_replica = get_host_replica(volume, host_id) volume.replicaRemove(name=host_replica["name"]) # Instead of waiting for timeout and lengthening the tests a significant # amount we can make sure the scheduling isn't working by making sure the # volume becomes Degraded and reports a scheduling error. volume = wait_for_volume_degraded(client, volume_name) wait_scheduling_failure(client, volume_name) # While there are three replicas that should exist to meet the Volume's # request, only two of those volumes should actually be Healthy. assert sum([ 1 for replica in volume["replicas"] if replica["running"] and replica["mode"] == "RW" ]) == 2 # Confirm that the final volume is an unscheduled volume. assert sum([1 for replica in volume["replicas"] if not replica["hostId"]]) == 1 # Three replicas in total should still exist. assert len(volume["replicas"]) == 3 check_volume_data(volume, data) cleanup_volume(client, volume)
def backupstore_test(client, host_id, volname, size): bv, b, snap2, data = create_backup(client, volname) # test restore restoreName = generate_volume_name() volume = client.create_volume(name=restoreName, size=size, numberOfReplicas=2, fromBackup=b["url"]) volume = common.wait_for_volume_restoration_completed(client, restoreName) volume = common.wait_for_volume_detached(client, restoreName) assert volume["name"] == restoreName assert volume["size"] == size assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["initialRestorationRequired"] is False volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, restoreName) check_volume_data(volume, data) volume = volume.detach() volume = common.wait_for_volume_detached(client, restoreName) bv.backupDelete(name=b["name"]) backups = bv.backupList() found = False for b in backups: if b["snapshotName"] == snap2["name"]: found = True break assert not found volume = wait_for_volume_status(client, volume["name"], "lastBackup", "") assert volume["lastBackupAt"] == "" client.delete(volume) volume = wait_for_volume_delete(client, restoreName)
def migration_rollback_test(clients, volume_name, backing_image=""): # NOQA client, volume, data = setup_migration_test(clients, volume_name, backing_image) host1, host2 = get_hosts_for_migration_test(clients) volume.attach(hostId=host1) volume = common.wait_for_volume_healthy(client, volume_name) volume.attach(hostId=host2) volume = common.wait_for_volume_migration_ready(client, volume_name) volume.detach(hostId=host2) volume = common.wait_for_volume_migration_node(client, volume_name, host1) volume.detach(hostId="") volume = common.wait_for_volume_detached(client, volume_name) # verify test data check_volume_data(client, volume_name, data) client.delete(volume) wait_for_volume_delete(client, volume_name)
def create_volume(client, vol_name, size, node_id, r_num): # NOQA volume = client.create_volume(name=vol_name, size=size, numberOfReplicas=r_num) assert volume["numberOfReplicas"] == r_num assert volume["frontend"] == "blockdev" volume = common.wait_for_volume_detached(client, vol_name) assert len(volume["replicas"]) == r_num assert volume["state"] == "detached" assert volume["created"] != "" volumeByName = client.by_id_volume(vol_name) assert volumeByName["name"] == volume["name"] assert volumeByName["size"] == volume["size"] assert volumeByName["numberOfReplicas"] == volume["numberOfReplicas"] assert volumeByName["state"] == volume["state"] assert volumeByName["created"] == volume["created"] volume.attach(hostId=node_id) volume = common.wait_for_volume_healthy(client, vol_name) return volume
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" data = write_volume_random_data(volume) snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) _, b = common.find_backup(client, volume_name, snap2["name"]) res_name = common.generate_volume_name() res_volume = client.create_volume(name=res_name, size=size, numberOfReplicas=2, fromBackup=b["url"]) res_volume = common.wait_for_volume_detached(client, res_name) res_volume = res_volume.attach(hostId=host_id) res_volume = common.wait_for_volume_healthy(client, res_name) check_volume_data(res_volume, data) snapshots = res_volume.snapshotList() # only the backup snapshot + volume-head assert len(snapshots) == 2 backup_snapshot = "" for snap in snapshots: if snap["name"] != "volume-head": backup_snapshot = snap["name"] assert backup_snapshot != "" res_volume.snapshotCreate() snapshots = res_volume.snapshotList() assert len(snapshots) == 3 res_volume.snapshotDelete(name=backup_snapshot) res_volume.snapshotPurge() snapshots = res_volume.snapshotList() assert len(snapshots) == 2 ha_rebuild_replica_test(client, res_name) res_volume = res_volume.detach() res_volume = common.wait_for_volume_detached(client, res_name) client.delete(res_volume) common.wait_for_volume_delete(client, res_name) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_replica_scheduler_update_minimal_available(client): # NOQA minimal_available_setting = client.by_id_setting( SETTING_STORAGE_MINIMAL_AVAILABLE_PERCENTAGE) old_minimal_setting = minimal_available_setting["value"] nodes = client.list_node() expect_node_disk = {} for node in nodes: disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == DEFAULT_DISK_PATH: expect_disk = disk expect_disk["fsid"] = fsid expect_node_disk[node["name"]] = expect_disk # set storage minimal available percentage to 100 # to test all replica couldn't be scheduled minimal_available_setting = client.update(minimal_available_setting, value="100") # wait for disks state nodes = client.list_node() for node in nodes: disks = node["disks"] for fsid, disk in disks.iteritems(): wait_for_disk_conditions(client, node["name"], fsid, DISK_CONDITION_SCHEDULABLE, CONDITION_STATUS_FALSE) lht_hostId = get_self_host_id() vol_name = common.generate_volume_name() volume = client.create_volume(name=vol_name, size=SIZE, numberOfReplicas=len(nodes)) volume = common.wait_for_volume_condition_scheduled(client, vol_name, "status", CONDITION_STATUS_FALSE) # set storage minimal available percentage to default value(10) minimal_available_setting = client.update(minimal_available_setting, value=old_minimal_setting) # wait for disks state nodes = client.list_node() for node in nodes: disks = node["disks"] for fsid, disk in disks.iteritems(): wait_for_disk_conditions(client, node["name"], fsid, DISK_CONDITION_SCHEDULABLE, CONDITION_STATUS_TRUE) # check volume status volume = common.wait_for_volume_condition_scheduled(client, vol_name, "status", CONDITION_STATUS_TRUE) volume = common.wait_for_volume_detached(client, vol_name) assert volume["state"] == "detached" assert volume["created"] != "" volume.attach(hostId=lht_hostId) volume = common.wait_for_volume_healthy(client, vol_name) nodes = client.list_node() node_hosts = [] for node in nodes: node_hosts.append(node["name"]) # check all replica should be scheduled to default disk for replica in volume["replicas"]: id = replica["hostId"] assert id != "" assert replica["running"] expect_disk = expect_node_disk[id] assert replica["diskID"] == expect_disk["fsid"] assert expect_disk["path"] in replica["dataPath"] node_hosts = filter(lambda x: x != id, node_hosts) assert len(node_hosts) == 0 # clean volume and disk cleanup_volume(client, vol_name)
def test_replica_scheduler_just_under_over_provisioning(client): # NOQA over_provisioning_setting = client.by_id_setting( SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE) old_provisioning_setting = over_provisioning_setting["value"] # set storage over provisioning percentage to 100 over_provisioning_setting = client.update(over_provisioning_setting, value="100") lht_hostId = get_self_host_id() nodes = client.list_node() expect_node_disk = {} max_size_array = [] for node in nodes: disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == DEFAULT_DISK_PATH: expect_disk = disk expect_disk["fsid"] = fsid expect_node_disk[node["name"]] = expect_disk max_size_array.append(disk["storageMaximum"]) disk["storageReserved"] = 0 update_disks = get_update_disks(disks) node = node.diskUpdate(disks=update_disks) disks = node["disks"] for fsid, disk in disks.iteritems(): wait_for_disk_status(client, node["name"], fsid, "storageReserved", 0) max_size = min(max_size_array) # test just under over provisioning limit could be scheduled vol_name = common.generate_volume_name() volume = client.create_volume(name=vol_name, size=str(max_size), numberOfReplicas=len(nodes)) volume = common.wait_for_volume_condition_scheduled(client, vol_name, "status", CONDITION_STATUS_TRUE) volume = common.wait_for_volume_detached(client, vol_name) assert volume["state"] == "detached" assert volume["created"] != "" volume.attach(hostId=lht_hostId) volume = common.wait_for_volume_healthy(client, vol_name) nodes = client.list_node() node_hosts = [] for node in nodes: node_hosts.append(node["name"]) # check all replica should be scheduled to default disk for replica in volume["replicas"]: id = replica["hostId"] assert id != "" assert replica["running"] expect_disk = expect_node_disk[id] assert replica["diskID"] == expect_disk["fsid"] assert expect_disk["path"] in replica["dataPath"] node_hosts = filter(lambda x: x != id, node_hosts) assert len(node_hosts) == 0 # clean volume and disk cleanup_volume(client, vol_name) client.update(over_provisioning_setting, value=old_provisioning_setting)
def test_replica_scheduler_too_large_volume_fit_any_disks(client): # NOQA nodes = client.list_node() lht_hostId = get_self_host_id() expect_node_disk = {} for node in nodes: disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == DEFAULT_DISK_PATH: expect_disk = disk expect_disk["fsid"] = fsid expect_node_disk[node["name"]] = expect_disk disk["storageReserved"] = disk["storageMaximum"] update_disks = get_update_disks(disks) node.diskUpdate(disks=update_disks) # volume is too large to fill into any disks vol_name = common.generate_volume_name() volume = client.create_volume(name=vol_name, size=str(4*Gi), numberOfReplicas=len(nodes)) volume = common.wait_for_volume_condition_scheduled(client, vol_name, "status", CONDITION_STATUS_FALSE) # reduce StorageReserved of each default disk nodes = client.list_node() for node in nodes: disks = node["disks"] update_disks = get_update_disks(disks) for disk in update_disks: disk["storageReserved"] = 0 node = node.diskUpdate(disks=update_disks) disks = node["disks"] for fsid, disk in disks.iteritems(): wait_for_disk_status(client, node["name"], fsid, "storageReserved", 0) # check volume status volume = common.wait_for_volume_condition_scheduled(client, vol_name, "status", CONDITION_STATUS_TRUE) volume = common.wait_for_volume_detached(client, vol_name) assert volume["state"] == "detached" assert volume["created"] != "" volume.attach(hostId=lht_hostId) volume = common.wait_for_volume_healthy(client, vol_name) nodes = client.list_node() node_hosts = [] for node in nodes: node_hosts.append(node["name"]) # check all replica should be scheduled to default disk for replica in volume["replicas"]: id = replica["hostId"] assert id != "" assert replica["running"] expect_disk = expect_node_disk[id] assert replica["diskID"] == expect_disk["fsid"] assert expect_disk["path"] in replica["dataPath"] node_hosts = filter(lambda x: x != id, node_hosts) assert len(node_hosts) == 0 # clean volume and disk cleanup_volume(client, vol_name)