def volume_iscsi_basic_test(clients, volume_name, base_image=""): # NOQA # get a random client for host_id, client in clients.iteritems(): break volume = create_and_check_volume(client, volume_name, 3, SIZE, base_image, "iscsi") volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == volume["name"] assert volumes[0]["size"] == volume["size"] assert volumes[0]["numberOfReplicas"] == volume["numberOfReplicas"] assert volumes[0]["state"] == volume["state"] assert volumes[0]["created"] == volume["created"] assert volumes[0]["frontend"] == "iscsi" endpoint = get_volume_endpoint(volumes[0]) assert endpoint.startswith("iscsi://") try: dev = iscsi_login(endpoint) volume_rw_test(dev) finally: iscsi_logout(endpoint) cleanup_volume(client, volume)
def create_host_disk(client, vol_name, size, node_id): # NOQA # create a single replica volume and attach it to node volume = create_volume(client, vol_name, size, node_id, 1) # prepare the disk in the host filesystem disk_path = common.prepare_host_disk(get_volume_endpoint(volume), volume["name"]) return disk_path
def create_host_disk(client, vol_name, size, node_id): # NOQA # create a single replica volume and attach it to node volume = create_volume(client, vol_name, size, node_id, 1) # prepare the disk in the host filesystem disk_path = common.prepare_host_disk(get_volume_endpoint(volume), volume["name"]) return disk_path
def volume_iscsi_basic_test(clients, volume_name, base_image=""): # NOQA # get a random client for host_id, client in clients.iteritems(): break volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3, frontend="iscsi", baseImage=base_image) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 3 assert volume["frontend"] == "iscsi" assert volume["baseImage"] == base_image volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == 3 assert volume["state"] == "detached" assert volume["created"] != "" volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == volume["name"] assert volumes[0]["size"] == volume["size"] assert volumes[0]["numberOfReplicas"] == volume["numberOfReplicas"] assert volumes[0]["state"] == volume["state"] assert volumes[0]["created"] == volume["created"] assert volumes[0]["frontend"] == "iscsi" endpoint = get_volume_endpoint(volumes[0]) assert endpoint.startswith("iscsi://") try: dev = iscsi_login(endpoint) volume_rw_test(dev) finally: iscsi_logout(endpoint) volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_volume_scheduling_failure(clients, volume_name): # NOQA ''' Test fail to schedule by disable scheduling for all the nodes Also test cannot attach a scheduling failed volume ''' client = get_random_client(clients) nodes = client.list_node() assert len(nodes) > 0 for node in nodes: node = client.update(node, allowScheduling=False) node = common.wait_for_node_update(client, node["id"], "allowScheduling", False) volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3) volume = common.wait_for_volume_condition_scheduled(client, volume_name, "status", CONDITION_STATUS_FALSE) volume = common.wait_for_volume_detached(client, volume_name) self_node = get_self_host_id() with pytest.raises(Exception) as e: volume.attach(hostId=self_node) assert "not scheduled" in str(e.value) for node in nodes: node = client.update(node, allowScheduling=True) node = common.wait_for_node_update(client, node["id"], "allowScheduling", True) volume = common.wait_for_volume_condition_scheduled(client, volume_name, "status", CONDITION_STATUS_TRUE) volume = common.wait_for_volume_detached(client, volume_name) volume = volume.attach(hostId=self_node) volume = common.wait_for_volume_healthy(client, volume_name) endpoint = get_volume_endpoint(volume) assert endpoint != "" volume_rw_test(endpoint) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name)
def ha_rebuild_replica_test(client, volname): # NOQA volume = client.by_id_volume(volname) assert get_volume_endpoint(volume) == DEV_PATH + volname assert len(volume["replicas"]) == 2 replica0 = volume["replicas"][0] assert replica0["name"] != "" replica1 = volume["replicas"][1] assert replica1["name"] != "" data = write_volume_random_data(volume) volume = volume.replicaRemove(name=replica0["name"]) # wait until we saw a replica starts rebuilding new_replica_found = False for i in range(RETRY_COUNTS): v = client.by_id_volume(volname) for r in v["replicas"]: if r["name"] != replica0["name"] and \ r["name"] != replica1["name"]: new_replica_found = True break if new_replica_found: break time.sleep(RETRY_INTERVAL) assert new_replica_found volume = common.wait_for_volume_healthy(client, volname) volume = client.by_id_volume(volname) assert volume["state"] == common.VOLUME_STATE_ATTACHED assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY assert len(volume["replicas"]) >= 2 found = False for replica in volume["replicas"]: if replica["name"] == replica1["name"]: found = True break assert found check_volume_data(volume, data)
def ha_rebuild_replica_test(client, volname): # NOQA volume = client.by_id_volume(volname) assert get_volume_endpoint(volume) == DEV_PATH + volname assert len(volume["replicas"]) == 2 replica0 = volume["replicas"][0] assert replica0["name"] != "" replica1 = volume["replicas"][1] assert replica1["name"] != "" data = write_volume_random_data(volume) volume = volume.replicaRemove(name=replica0["name"]) # wait until we saw a replica starts rebuilding new_replica_found = False for i in range(RETRY_COUNTS): v = client.by_id_volume(volname) for r in v["replicas"]: if r["name"] != replica0["name"] and \ r["name"] != replica1["name"]: new_replica_found = True break if new_replica_found: break time.sleep(RETRY_INTERVAL) assert new_replica_found volume = common.wait_for_volume_healthy(client, volname) volume = client.by_id_volume(volname) assert volume["state"] == common.VOLUME_STATE_ATTACHED assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY assert len(volume["replicas"]) >= 2 found = False for replica in volume["replicas"]: if replica["name"] == replica1["name"]: found = True break assert found check_volume_data(volume, data)
def test_xfs_pv_existing_volume(client, core_api, pod_manifest): # NOQA """ Test create PV with existing XFS filesystem 1. Create a volume 2. Create PV/PVC for the existing volume, specify `xfs` as filesystem 3. Attach the volume to the current node. 4. Format it to `xfs` 5. Create a POD using the volume FIXME: We should write data in step 4 and validate the data in step 5, make sure the disk won't be reformatted """ volume_name = generate_volume_name() volume = create_and_check_volume(client, volume_name) create_pv_for_volume(client, core_api, volume, volume_name, "xfs") create_pvc_for_volume(client, core_api, volume, volume_name) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) cmd = ['mkfs.xfs', get_volume_endpoint(volume)] subprocess.check_call(cmd) volume = volume.detach() volume = wait_for_volume_detached(client, volume_name) pod_manifest['spec']['volumes'] = [{ "name": "pod-data", "persistentVolumeClaim": { "claimName": volume_name } }] create_and_wait_pod(core_api, pod_manifest)
def volume_basic_test(clients, volume_name, base_image=""): # NOQA num_hosts = len(clients) num_replicas = 3 # get a random client for host_id, client in clients.iteritems(): break with pytest.raises(Exception): volume = client.create_volume(name="wrong_volume-name-1.0", size=SIZE, numberOfReplicas=2) volume = client.create_volume(name="wrong_volume-name", size=SIZE, numberOfReplicas=2) volume = client.create_volume(name="wrong_volume-name", size=SIZE, numberOfReplicas=2, frontend="invalid_frontend") volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=num_replicas, baseImage=base_image) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == num_replicas assert volume["frontend"] == "blockdev" assert volume["baseImage"] == base_image volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == num_replicas assert volume["state"] == "detached" assert volume["created"] != "" def validate_volume_basic(expected, actual): assert actual["name"] == expected["name"] assert actual["size"] == expected["size"] assert actual["numberOfReplicas"] == expected["numberOfReplicas"] assert actual["frontend"] == "blockdev" assert actual["baseImage"] == base_image assert actual["state"] == expected["state"] assert actual["created"] == expected["created"] volumes = client.list_volume() assert len(volumes) == 1 validate_volume_basic(volume, volumes[0]) volumeByName = client.by_id_volume(volume_name) validate_volume_basic(volume, volumeByName) lht_hostId = get_self_host_id() volume.attach(hostId=lht_hostId) volume = common.wait_for_volume_healthy(client, volume_name) volumeByName = client.by_id_volume(volume_name) validate_volume_basic(volume, volumeByName) assert get_volume_endpoint(volumeByName) == DEV_PATH + volume_name # validate soft anti-affinity hosts = {} for replica in volume["replicas"]: id = replica["hostId"] assert id != "" hosts[id] = True if num_hosts >= num_replicas: assert len(hosts) == num_replicas else: assert len(hosts) == num_hosts volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == volume["name"] assert volumes[0]["size"] == volume["size"] assert volumes[0]["numberOfReplicas"] == volume["numberOfReplicas"] assert volumes[0]["state"] == volume["state"] assert volumes[0]["created"] == volume["created"] assert get_volume_endpoint(volumes[0]) == DEV_PATH + volume_name volume = client.by_id_volume(volume_name) assert get_volume_endpoint(volume) == DEV_PATH + volume_name volume_rw_test(get_volume_endpoint(volume)) volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_node_delete_umount_disks(client): # NOQA # create test disks for node disk_volume_name = 'vol-disk-1' lht_hostId = get_self_host_id() node = client.by_id_node(lht_hostId) disks = node["disks"] disk_path1 = create_host_disk(client, disk_volume_name, str(Gi), lht_hostId) disk1 = { "path": disk_path1, "allowScheduling": True, "storageReserved": SMALL_DISK_SIZE } update_disk = get_update_disks(disks) for disk in update_disk: disk["allowScheduling"] = False # add new disk for node update_disk.append(disk1) # save disks to node node = node.diskUpdate(disks=update_disk) node = common.wait_for_disk_update(client, lht_hostId, len(update_disk)) assert len(node["disks"]) == len(update_disk) node = client.by_id_node(lht_hostId) assert len(node["disks"]) == len(update_disk) disks = node["disks"] # wait for node controller to update disk status for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling", True) wait_for_disk_status(client, lht_hostId, fsid, "storageReserved", SMALL_DISK_SIZE) free, total = common.get_host_disk_size(disk_path1) wait_for_disk_status(client, lht_hostId, fsid, "storageAvailable", free) wait_for_disk_status(client, lht_hostId, fsid, "storageMaximum", total) node = client.by_id_node(lht_hostId) disks = node["disks"] for key, disk in disks.iteritems(): if disk["path"] == disk_path1: assert disk["allowScheduling"] assert disk["storageReserved"] == SMALL_DISK_SIZE assert disk["storageScheduled"] == 0 free, total = common.get_host_disk_size(disk_path1) assert disk["storageMaximum"] == total assert disk["storageAvailable"] == free conditions = disk["conditions"] assert conditions[DISK_CONDITION_READY]["status"] == \ CONDITION_STATUS_TRUE assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \ CONDITION_STATUS_TRUE else: assert not disk["allowScheduling"] # create a volume nodes = client.list_node() vol_name = common.generate_volume_name() volume = create_volume(client, vol_name, str(SMALL_DISK_SIZE), lht_hostId, len(nodes)) replicas = volume["replicas"] for replica in replicas: id = replica["hostId"] assert id != "" assert replica["running"] if id == lht_hostId: assert replica["dataPath"].startswith(disk_path1) # umount the disk mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name) common.umount_disk(mount_path) # wait for update node status node = client.by_id_node(lht_hostId) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling", False) wait_for_disk_status(client, lht_hostId, fsid, "storageMaximum", 0) wait_for_disk_conditions(client, lht_hostId, fsid, DISK_CONDITION_READY, CONDITION_STATUS_FALSE) # check result node = client.by_id_node(lht_hostId) disks = node["disks"] update_disks = [] for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: assert not disk["allowScheduling"] assert disk["storageMaximum"] == 0 assert disk["storageAvailable"] == 0 assert disk["storageReserved"] == SMALL_DISK_SIZE assert disk["storageScheduled"] == SMALL_DISK_SIZE conditions = disk["conditions"] assert conditions[DISK_CONDITION_READY]["status"] == \ CONDITION_STATUS_FALSE assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \ CONDITION_STATUS_FALSE else: conditions = disk["conditions"] assert conditions[DISK_CONDITION_READY]["status"] == \ CONDITION_STATUS_TRUE assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \ CONDITION_STATUS_TRUE update_disks.append(disk) # delete umount disk exception with pytest.raises(Exception) as e: node.diskUpdate(disks=update_disks) assert "disable the disk" in str(e.value) # update other disks disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] != disk_path1: disk["allowScheduling"] = True test_update = get_update_disks(disks) node = node.diskUpdate(disks=test_update) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] != disk_path1: wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling", True) node = client.by_id_node(lht_hostId) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] != disk_path1: assert disk["allowScheduling"] # mount the disk back mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name) disk_volume = client.by_id_volume(disk_volume_name) dev = get_volume_endpoint(disk_volume) common.mount_disk(dev, mount_path) # wait for update node status node = client.by_id_node(lht_hostId) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling", False) wait_for_disk_conditions(client, lht_hostId, fsid, DISK_CONDITION_READY, CONDITION_STATUS_TRUE) # check result node = client.by_id_node(lht_hostId) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: free, total = common.get_host_disk_size(disk_path1) assert not disk["allowScheduling"] assert disk["storageMaximum"] == total assert disk["storageAvailable"] == free assert disk["storageReserved"] == SMALL_DISK_SIZE assert disk["storageScheduled"] == SMALL_DISK_SIZE conditions = disk["conditions"] assert conditions[DISK_CONDITION_READY]["status"] == \ CONDITION_STATUS_TRUE assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \ CONDITION_STATUS_TRUE else: conditions = disk["conditions"] assert conditions[DISK_CONDITION_READY]["status"] == \ CONDITION_STATUS_TRUE assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \ CONDITION_STATUS_TRUE # delete volume and umount disk cleanup_volume(client, vol_name) mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name) common.umount_disk(mount_path) # wait for update node status node = client.by_id_node(lht_hostId) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling", False) wait_for_disk_status(client, lht_hostId, fsid, "storageScheduled", 0) wait_for_disk_status(client, lht_hostId, fsid, "storageMaximum", 0) # test delete the umount disk node = client.by_id_node(lht_hostId) node.diskUpdate(disks=update_disks) node = common.wait_for_disk_update(client, lht_hostId, len(update_disks)) assert len(node["disks"]) == len(update_disks) cmd = ['rm', '-r', mount_path] subprocess.check_call(cmd)
def test_node_delete_umount_disks(client): # NOQA # create test disks for node disk_volume_name = 'vol-disk-1' lht_hostId = get_self_host_id() node = client.by_id_node(lht_hostId) disks = node["disks"] disk_path1 = create_host_disk(client, disk_volume_name, str(Gi), lht_hostId) disk1 = {"path": disk_path1, "allowScheduling": True, "storageReserved": SMALL_DISK_SIZE} update_disk = get_update_disks(disks) for disk in update_disk: disk["allowScheduling"] = False # add new disk for node update_disk.append(disk1) # save disks to node node = node.diskUpdate(disks=update_disk) node = common.wait_for_disk_update(client, lht_hostId, len(update_disk)) assert len(node["disks"]) == len(update_disk) node = client.by_id_node(lht_hostId) assert len(node["disks"]) == len(update_disk) disks = node["disks"] # wait for node controller to update disk status for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling", True) wait_for_disk_status(client, lht_hostId, fsid, "storageReserved", SMALL_DISK_SIZE) free, total = common.get_host_disk_size(disk_path1) wait_for_disk_status(client, lht_hostId, fsid, "storageAvailable", free) wait_for_disk_status(client, lht_hostId, fsid, "storageMaximum", total) node = client.by_id_node(lht_hostId) disks = node["disks"] for key, disk in disks.iteritems(): if disk["path"] == disk_path1: assert disk["allowScheduling"] assert disk["storageReserved"] == SMALL_DISK_SIZE assert disk["storageScheduled"] == 0 free, total = common.get_host_disk_size(disk_path1) assert disk["storageMaximum"] == total assert disk["storageAvailable"] == free conditions = disk["conditions"] assert conditions[DISK_CONDITION_READY]["status"] == \ CONDITION_STATUS_TRUE assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \ CONDITION_STATUS_TRUE else: assert not disk["allowScheduling"] # create a volume nodes = client.list_node() vol_name = common.generate_volume_name() volume = create_volume(client, vol_name, str(SMALL_DISK_SIZE), lht_hostId, len(nodes)) replicas = volume["replicas"] for replica in replicas: id = replica["hostId"] assert id != "" assert replica["running"] if id == lht_hostId: assert replica["dataPath"].startswith(disk_path1) # umount the disk mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name) common.umount_disk(mount_path) # wait for update node status node = client.by_id_node(lht_hostId) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling", False) wait_for_disk_status(client, lht_hostId, fsid, "storageMaximum", 0) wait_for_disk_conditions(client, lht_hostId, fsid, DISK_CONDITION_READY, CONDITION_STATUS_FALSE) # check result node = client.by_id_node(lht_hostId) disks = node["disks"] update_disks = [] for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: assert not disk["allowScheduling"] assert disk["storageMaximum"] == 0 assert disk["storageAvailable"] == 0 assert disk["storageReserved"] == SMALL_DISK_SIZE assert disk["storageScheduled"] == SMALL_DISK_SIZE conditions = disk["conditions"] assert conditions[DISK_CONDITION_READY]["status"] == \ CONDITION_STATUS_FALSE assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \ CONDITION_STATUS_FALSE else: conditions = disk["conditions"] assert conditions[DISK_CONDITION_READY]["status"] == \ CONDITION_STATUS_TRUE assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \ CONDITION_STATUS_TRUE update_disks.append(disk) # delete umount disk exception with pytest.raises(Exception) as e: node.diskUpdate(disks=update_disks) assert "disable the disk" in str(e.value) # update other disks disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] != disk_path1: disk["allowScheduling"] = True test_update = get_update_disks(disks) node = node.diskUpdate(disks=test_update) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] != disk_path1: wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling", True) node = client.by_id_node(lht_hostId) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] != disk_path1: assert disk["allowScheduling"] # mount the disk back mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name) disk_volume = client.by_id_volume(disk_volume_name) dev = get_volume_endpoint(disk_volume) common.mount_disk(dev, mount_path) # wait for update node status node = client.by_id_node(lht_hostId) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling", False) wait_for_disk_conditions(client, lht_hostId, fsid, DISK_CONDITION_READY, CONDITION_STATUS_TRUE) # check result node = client.by_id_node(lht_hostId) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: free, total = common.get_host_disk_size(disk_path1) assert not disk["allowScheduling"] assert disk["storageMaximum"] == total assert disk["storageAvailable"] == free assert disk["storageReserved"] == SMALL_DISK_SIZE assert disk["storageScheduled"] == SMALL_DISK_SIZE conditions = disk["conditions"] assert conditions[DISK_CONDITION_READY]["status"] == \ CONDITION_STATUS_TRUE assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \ CONDITION_STATUS_TRUE else: conditions = disk["conditions"] assert conditions[DISK_CONDITION_READY]["status"] == \ CONDITION_STATUS_TRUE assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \ CONDITION_STATUS_TRUE # delete volume and umount disk cleanup_volume(client, vol_name) mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name) common.umount_disk(mount_path) # wait for update node status node = client.by_id_node(lht_hostId) disks = node["disks"] for fsid, disk in disks.iteritems(): if disk["path"] == disk_path1: wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling", False) wait_for_disk_status(client, lht_hostId, fsid, "storageScheduled", 0) wait_for_disk_status(client, lht_hostId, fsid, "storageMaximum", 0) # test delete the umount disk node = client.by_id_node(lht_hostId) node.diskUpdate(disks=update_disks) node = common.wait_for_disk_update(client, lht_hostId, len(update_disks)) assert len(node["disks"]) == len(update_disks) cmd = ['rm', '-r', mount_path] subprocess.check_call(cmd)
def restore_and_check_random_backup(client, core_api, volume_name, pod_name, snapshots_md5sum): # NOQA res_volume_name = volume_name + '-restore' host_id = get_self_host_id() snap_data = get_random_backup_snapshot_data(snapshots_md5sum) if snap_data is None: print("skipped, no recorded backup found", end=" ") return backup_url = snap_data.backup_url client.create_volume(name=res_volume_name, size=VOLUME_SIZE, fromBackup=backup_url) wait_for_volume_restoration_completed(client, res_volume_name) wait_for_volume_detached(client, res_volume_name) res_volume = client.by_id_volume(res_volume_name) res_volume.attach(hostId=host_id) res_volume = wait_for_volume_healthy(client, res_volume_name) dev = get_volume_endpoint(res_volume) mount_path = os.path.join(DIRECTORY_PATH, res_volume_name) command = ['mkdir', '-p', mount_path] subprocess.check_call(command) mount_disk(dev, mount_path) datafile_name = get_data_filename(pod_name) datafile_path = os.path.join(mount_path, datafile_name) command = ['md5sum', datafile_path] output = subprocess.check_output(command) bkp_data_md5sum = output.split()[0].decode('utf-8') bkp_checksum_ok = False if snap_data.data_md5sum == bkp_data_md5sum: bkp_checksum_ok = True umount_disk(mount_path) command = ['rmdir', mount_path] subprocess.check_call(command) res_volume = client.by_id_volume(res_volume_name) res_volume.detach() wait_for_volume_detached(client, res_volume_name) delete_and_wait_longhorn(client, res_volume_name) assert bkp_checksum_ok
def backing_image_content_test(client, volume_name_prefix, bi_name, bi_url): # NOQA """ Verify the content of the Backing Image is accessible and read-only for all volumes. 1. Create a backing image. (Done by the caller) 2. Create a Volume with the backing image set then attach it to host node. 3. Verify that the all disk states in the backing image are "downloaded". 4. Verify volume can be directly mounted and there is already data in the filesystem due to the backing image. 5. Verify the volume r/w. 6. Launch one more volume with the same backing image. 7. Verify the data content of the new volume is the same as the data in step 4. 5. Do cleanup. (Done by the caller) """ lht_host_id = get_self_host_id() volume_name1 = volume_name_prefix + "-1" volume1 = create_and_check_volume( client, volume_name1, 3, str(BACKING_IMAGE_EXT4_SIZE), bi_name) volume1.attach(hostId=lht_host_id) volume1 = wait_for_volume_healthy(client, volume_name1) assert volume1.backingImage == bi_name assert volume1.size == str(BACKING_IMAGE_EXT4_SIZE) backing_image = client.by_id_backing_image(bi_name) assert backing_image.imageURL == bi_url assert not backing_image.deletionTimestamp assert len(backing_image.diskStateMap) == 3 for disk_id, state in iter(backing_image.diskStateMap.items()): assert state == "downloaded" # Since there is already a filesystem with data in the backing image, # we can directly mount and access the volume without `mkfs`. dev1 = get_volume_endpoint(volume1) mount_path1 = os.path.join(DIRECTORY_PATH, volume_name1) mount_disk(dev1, mount_path1) output1 = subprocess.check_output(["ls", mount_path1]) # The following random write may crash the filesystem of volume1, # need to umount it here cleanup_host_disk(volume_name1) # Verify r/w for the volume with a backing image data = write_volume_random_data(volume1) check_volume_data(volume1, data) volume_name2 = volume_name_prefix + "-2" volume2 = create_and_check_volume( client, volume_name2, 3, str(BACKING_IMAGE_EXT4_SIZE), bi_name) volume2.attach(hostId=lht_host_id) volume2 = wait_for_volume_healthy(client, volume_name2) assert volume1.backingImage == bi_name assert volume1.size == str(BACKING_IMAGE_EXT4_SIZE) dev2 = get_volume_endpoint(volume2) mount_path2 = os.path.join(DIRECTORY_PATH, volume_name2) mount_disk(dev2, mount_path2) output2 = subprocess.check_output(["ls", mount_path2]) # The output is the content of the backing image, which should keep # unchanged assert output2 == output1 cleanup_host_disk(volume_name2)
def ha_simple_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == size assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["created"] != "" assert volume["baseImage"] == base_image host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) volume = client.by_id_volume(volume_name) assert get_volume_endpoint(volume) == DEV_PATH + volume_name assert len(volume["replicas"]) == 2 replica0 = volume["replicas"][0] assert replica0["name"] != "" replica1 = volume["replicas"][1] assert replica1["name"] != "" data = write_volume_random_data(volume) volume = volume.replicaRemove(name=replica0["name"]) # wait until we saw a replica starts rebuilding new_replica_found = False for i in range(RETRY_COUNTS): v = client.by_id_volume(volume_name) for r in v["replicas"]: if r["name"] != replica0["name"] and \ r["name"] != replica1["name"]: new_replica_found = True break if new_replica_found: break time.sleep(RETRY_ITERVAL) assert new_replica_found volume = common.wait_for_volume_healthy(client, volume_name) volume = client.by_id_volume(volume_name) assert volume["state"] == common.VOLUME_STATE_ATTACHED assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY assert len(volume["replicas"]) >= 2 found = False for replica in volume["replicas"]: if replica["name"] == replica1["name"]: found = True break assert found check_volume_data(volume, data) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0