def csi_mount_test( client, core_api, csi_pv, pvc, pod, # NOQA volume_size, base_image=""): # NOQA pod_name = 'csi-mount-test' pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])] pvc['spec']['volumeName'] = csi_pv['metadata']['name'] update_storageclass_references(CSI_PV_TEST_STORAGE_NAME, csi_pv, pvc) create_pv_storage(core_api, client, csi_pv, pvc, base_image) create_and_wait_pod(core_api, pod) volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == csi_pv['metadata']['name'] assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == \ int(csi_pv['spec']['csi']['volumeAttributes']["numberOfReplicas"]) assert volumes[0]["state"] == "attached" assert volumes[0]["baseImage"] == base_image
def test_flexvolume_mount(client, core_api, flexvolume, pod): # NOQA """ Test that a statically defined volume can be created, mounted, unmounted, and deleted properly on the Kubernetes cluster. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ pod_name = 'flexvolume-mount-test' pod['metadata']['name'] = pod_name pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \ flexvolume['name'] pod['spec']['volumes'] = [flexvolume] volume_size = DEFAULT_VOLUME_SIZE * Gi create_and_wait_pod(core_api, pod) volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == flexvolume['name'] assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == int( flexvolume["flexVolume"]["options"]["numberOfReplicas"]) assert volumes[0]["state"] == "attached"
def ha_simple_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == size assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["created"] != "" assert volume["baseImage"] == base_image host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) ha_rebuild_replica_test(client, volume_name) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_ha_prohibit_deleting_last_replica(client, volume_name): # NOQA volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=1) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 1 assert volume["state"] == "detached" assert volume["created"] != "" host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 1 replica0 = volume["replicas"][0] with pytest.raises(Exception) as e: volume.replicaRemove(name=replica0["name"]) assert "no other healthy replica available" in str(e.value) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_csi_mount(client, core_api, csi_pv, pvc, pod): # NOQA """ Test that a statically defined CSI volume can be created, mounted, unmounted, and deleted properly on the Kubernetes cluster. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ pod_name = 'csi-mount-test' pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])] pvc['spec']['volumeName'] = csi_pv['metadata']['name'] volume_size = DEFAULT_VOLUME_SIZE * Gi create_pv_storage(core_api, client, csi_pv, pvc) create_and_wait_pod(core_api, pod) volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == csi_pv['metadata']['name'] assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == \ int(csi_pv['spec']['csi']['volumeAttributes']["numberOfReplicas"]) assert volumes[0]["state"] == "attached"
def test_statefulset_mount(client, core_api, storage_class, statefulset): # NOQA """ Tests that volumes provisioned for a StatefulSet can be properly created, mounted, unmounted, and deleted on the Kubernetes cluster. """ statefulset_name = 'statefulset-mount-test' update_test_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) pod_info = get_statefulset_pod_info(core_api, statefulset) volumes = client.list_volume() assert len(volumes) == statefulset['spec']['replicas'] for v in volumes: # Workaround for checking volume name since they differ per pod. found = False for pod in pod_info: if v['name'] == pod['pv_name']: found = True break assert found pod_info.remove(pod) assert v['size'] == str(DEFAULT_VOLUME_SIZE * Gi) assert v['numberOfReplicas'] == \ int(storage_class['parameters']['numberOfReplicas']) assert v['state'] == 'attached' # Confirm that we've iterated through all the volumes. assert len(pod_info) == 0
def test_provisioner_mount(client, core_api, storage_class, pvc, pod): # NOQA """ Test that a StorageClass provisioned volume can be created, mounted, unmounted, and deleted properly on the Kubernetes cluster. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ # Prepare pod and volume specs. pod_name = 'provisioner-mount-test' pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])] pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME volume_size = DEFAULT_VOLUME_SIZE * Gi create_storage(core_api, storage_class, pvc) create_and_wait_pod(core_api, pod) pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name']) # Confirm that the volume has all the correct parameters we gave it. volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == pvc_volume_name assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == \ int(storage_class['parameters']['numberOfReplicas']) assert volumes[0]["state"] == "attached"
def test_flexvolume_tags(client, core_api, node_default_tags, flexvolume, pod): # NOQA """ Test that the FlexVolume provisioner is able to handle diskSelectors and nodeSelectors properly. """ pod_name = 'flexvolume-tag-test' tag_spec = { "disk": ["ssd", "nvme"], "expected": 1, "node": ["storage", "main"] } volume_size = DEFAULT_VOLUME_SIZE * Gi flexvolume["flexvolume"]["options"]["diskSelector"] = "ssd,nvme" flexvolume["flexvolume"]["options"]["nodeSelector"] = "storage,main" pod['metadata']['name'] = pod_name pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \ flexvolume['name'] pod['spec']['volumes'] = [flexvolume] create_and_wait_pod(core_api, pod) volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == flexvolume['name'] assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == int( flexvolume["flexVolume"]["options"]["numberOfReplicas"]) assert volumes[0]["state"] == "attached" check_volume_replicas(volumes[0], tag_spec, node_default_tags)
def test_provisioner_params(client, core_api, storage_class, pvc, pod): # NOQA """ Test that substituting different StorageClass parameters is reflected in the resulting PersistentVolumeClaim. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ # Prepare pod and volume specs. pod_name = 'provisioner-params-test' volume_size = 2 * Gi pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])] pvc['spec']['resources']['requests']['storage'] = \ size_to_string(volume_size) pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME storage_class['parameters'] = { 'numberOfReplicas': '2', 'staleReplicaTimeout': '20' } create_storage(core_api, storage_class, pvc) create_and_wait_pod(core_api, pod) pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name']) # Confirm that the volume has all the correct parameters we gave it. volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == pvc_volume_name assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == \ int(storage_class['parameters']['numberOfReplicas']) assert volumes[0]["state"] == "attached"
def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): # NOQA break # error when creating volume with duplicate jobs with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1() + create_jobs1()) assert "duplicate job" in str(e.value) client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1()) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_provisioner_mount(client, core_api, storage_class, pvc, pod): # NOQA """ Test that a StorageClass provisioned volume can be created, mounted, unmounted, and deleted properly on the Kubernetes cluster. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ # Prepare pod and volume specs. pod_name = 'provisioner-mount-test' pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [ create_pvc_spec(pvc['metadata']['name']) ] pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME volume_size = DEFAULT_VOLUME_SIZE * Gi create_storage(core_api, storage_class, pvc) create_and_wait_pod(core_api, pod) pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name']) # Confirm that the volume has all the correct parameters we gave it. volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == pvc_volume_name assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == \ int(storage_class['parameters']['numberOfReplicas']) assert volumes[0]["state"] == "attached"
def test_statefulset_mount(client, core_api, storage_class, statefulset): # NOQA """ Tests that volumes provisioned for a StatefulSet can be properly created, mounted, unmounted, and deleted on the Kubernetes cluster. """ statefulset_name = 'statefulset-mount-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) pod_info = get_statefulset_pod_info(core_api, statefulset) volumes = client.list_volume() assert len(volumes) == statefulset['spec']['replicas'] for v in volumes: # Workaround for checking volume name since they differ per pod. found = False for pod in pod_info: if v['name'] == pod['pv_name']: found = True break assert found pod_info.remove(pod) assert v['size'] == str(DEFAULT_VOLUME_SIZE * Gi) assert v['numberOfReplicas'] == \ int(storage_class['parameters']['numberOfReplicas']) assert v['state'] == 'attached' # Confirm that we've iterated through all the volumes. assert len(pod_info) == 0
def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): # NOQA break # error when creating volume with duplicate jobs with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1() + create_jobs1()) assert "duplicate job" in str(e.value) client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1()) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_tag_scheduling_failure(client, node_default_tags): # NOQA """ Test that scheduling fails if no Nodes/Disks with the requested Tags are available. Case 1: Validate that if specifying nonexist tags in volume, API call will fail. Case 2: 1. Specify existing but no node or disk can unsatisfied tags. 2. Validate the volume will failed the scheduling """ invalid_tag_cases = [ # Only one Disk Tag exists. { "disk": ["doesnotexist", "ssd"], "node": [] }, # Only one Node Tag exists. { "disk": [], "node": ["doesnotexist", "main"] } ] for tags in invalid_tag_cases: volume_name = generate_volume_name() # NOQA with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3, diskSelector=tags["disk"], nodeSelector=tags["node"]) assert "does not exist" in str(e.value) unsatisfied_tag_cases = [{ "disk": [], "node": ["main", "fallback"] }, { "disk": ["ssd", "m2"], "node": [] }] for tags in unsatisfied_tag_cases: volume_name = generate_volume_name() client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3, diskSelector=tags["disk"], nodeSelector=tags["node"]) volume = wait_for_volume_detached(client, volume_name) assert volume.diskSelector == tags["disk"] assert volume.nodeSelector == tags["node"] wait_scheduling_failure(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume.name) volumes = client.list_volume() assert len(volumes) == 0
def csi_mount_test(client, core_api, csi_pv, pvc, pod_make, # NOQA volume_size, base_image=""): # NOQA create_and_wait_csi_pod('csi-mount-test', client, core_api, csi_pv, pvc, pod_make, base_image, "") volumes = client.list_volume().data assert len(volumes) == 1 assert volumes[0].name == csi_pv['metadata']['name'] assert volumes[0].size == str(volume_size) assert volumes[0].numberOfReplicas == \ int(csi_pv['spec']['csi']['volumeAttributes']["numberOfReplicas"]) assert volumes[0].state == "attached" assert volumes[0].baseImage == base_image
def csi_mount_test(client, core_api, csi_pv, pvc, pod_make, # NOQA volume_size, base_image=""): # NOQA create_and_wait_csi_pod('csi-mount-test', client, core_api, csi_pv, pvc, pod_make, base_image, "") volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == csi_pv['metadata']['name'] assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == \ int(csi_pv['spec']['csi']['volumeAttributes']["numberOfReplicas"]) assert volumes[0]["state"] == "attached" assert volumes[0]["baseImage"] == base_image
def test_ha_salvage(client, volume_name): # NOQA # get a random client volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["created"] != "" host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 2 replica0_name = volume["replicas"][0]["name"] replica1_name = volume["replicas"][1]["name"] data = write_random_data(volume["endpoint"]) common.k8s_delete_replica_pods_for_volume(volume_name) volume = common.wait_for_volume_faulted(client, volume_name) assert len(volume["replicas"]) == 2 assert volume["replicas"][0]["failedAt"] != "" assert volume["replicas"][1]["failedAt"] != "" volume.salvage(names=[replica0_name, replica1_name]) volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == 2 assert volume["replicas"][0]["failedAt"] == "" assert volume["replicas"][1]["failedAt"] == "" volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) check_data(volume["endpoint"], data) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_recurring_job_in_volume_creation(set_random_backupstore, client, volume_name): # NOQA """ Test create volume with recurring jobs 1. Create volume with recurring jobs though Longhorn API 2. Verify the recurring jobs run correctly """ host_id = get_self_host_id() # error when creating volume with duplicate jobs with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1() + create_jobs1()) assert "duplicate job" in str(e.value) client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1()) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) # wait until the beginning of an even minute wait_until_begin_of_an_even_minute() # wait until the 10th second of an even minute # to avoid writing data at the same time backup is taking time.sleep(10) write_volume_random_data(volume) time.sleep(150) # 2.5 minutes write_volume_random_data(volume) time.sleep(150) # 2.5 minutes check_jobs1_result(volume) volume = volume.detach(hostId="") common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_provisioner_tags(client, core_api, node_default_tags, storage_class, pvc, pod): # NOQA """ Test that a StorageClass can properly provision a volume with requested Tags. Test prerequisite: - set Replica Node Level Soft Anti-Affinity enabled 1. Use `node_default_tags` to add default tags to nodes. 2. Create a StorageClass with disk and node tag set. 3. Create PVC and Pod. 4. Verify the volume has the correct parameters and tags. """ replica_node_soft_anti_affinity_setting = \ client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY) client.update(replica_node_soft_anti_affinity_setting, value="true") # Prepare pod and volume specs. pod_name = 'provisioner-tags-test' tag_spec = { "disk": ["ssd", "nvme"], "expected": 1, "node": ["storage", "main"] } pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])] pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME storage_class['parameters']['diskSelector'] = 'ssd,nvme' storage_class['parameters']['nodeSelector'] = 'storage,main' volume_size = DEFAULT_VOLUME_SIZE * Gi create_storage(core_api, storage_class, pvc) create_and_wait_pod(core_api, pod) pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name']) # Confirm that the volume has all the correct parameters we gave it. volumes = client.list_volume() assert len(volumes) == 1 assert volumes.data[0].name == pvc_volume_name assert volumes.data[0].size == str(volume_size) assert volumes.data[0].numberOfReplicas == \ int(storage_class['parameters']['numberOfReplicas']) assert volumes.data[0].state == "attached" check_volume_replicas(volumes.data[0], tag_spec, node_default_tags)
def ha_salvage_test(client, volume_name, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["created"] != "" assert volume["baseImage"] == base_image host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) assert len(volume["replicas"]) == 2 replica0_name = volume["replicas"][0]["name"] replica1_name = volume["replicas"][1]["name"] data = write_volume_random_data(volume) common.k8s_delete_replica_pods_for_volume(volume_name) volume = common.wait_for_volume_faulted(client, volume_name) assert len(volume["replicas"]) == 2 assert volume["replicas"][0]["failedAt"] != "" assert volume["replicas"][1]["failedAt"] != "" volume.salvage(names=[replica0_name, replica1_name]) volume = common.wait_for_volume_detached(client, volume_name) assert len(volume["replicas"]) == 2 assert volume["replicas"][0]["failedAt"] == "" assert volume["replicas"][1]["failedAt"] == "" volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) check_volume_data(volume, data) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_recurring_job(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): # NOQA break volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) jobs = create_jobs1() volume.recurringUpdate(jobs=jobs) volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) job_backup2 = { "name": "backup2", "cron": "* * * * *", "task": "backup", "retain": 2 } volume.recurringUpdate(jobs=[jobs[0], job_backup2]) # 5 minutes time.sleep(300) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot["removed"] is False: count += 1 # 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head assert count == 6 volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def flexvolume_mount_test(client, core_api, flexvolume, pod, volume_size): # NOQA pod_name = 'flexvolume-mount-test' pod['metadata']['name'] = pod_name pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \ flexvolume['name'] pod['spec']['volumes'] = [flexvolume] create_and_wait_pod(core_api, pod) volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == flexvolume['name'] assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == int( flexvolume["flexVolume"]["options"]["numberOfReplicas"]) assert volumes[0]["state"] == "attached"
def flexvolume_mount_test(client, core_api, flexvolume, pod, volume_size): # NOQA pod_name = 'flexvolume-mount-test' pod['metadata']['name'] = pod_name pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \ flexvolume['name'] pod['spec']['volumes'] = [ flexvolume ] create_and_wait_pod(core_api, pod) volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == flexvolume['name'] assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == int( flexvolume["flexVolume"]["options"]["numberOfReplicas"]) assert volumes[0]["state"] == "attached"
def test_recurring_job(clients, volume_name): # NOQA for host_id, client in clients.iteritems(): # NOQA break volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) jobs = create_jobs1() volume.recurringUpdate(jobs=jobs) volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) job_backup2 = {"name": "backup2", "cron": "* * * * *", "task": "backup", "retain": 2} volume.recurringUpdate(jobs=[jobs[0], job_backup2]) # 5 minutes time.sleep(300) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot["removed"] is False: count += 1 # 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head assert count == 6 volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA """ Test create volume with recurring jobs 1. Create volume with recurring jobs though Longhorn API 2. Verify the recurring jobs run correctly """ for host_id, client in iter(clients.items()): # NOQA break set_random_backupstore(client) # error when creating volume with duplicate jobs with pytest.raises(Exception) as e: client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1() + create_jobs1()) assert "duplicate job" in str(e.value) client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2, recurringJobs=create_jobs1()) volume = common.wait_for_volume_detached(client, volume_name) volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_provisioner_params(client, core_api, storage_class, pvc, pod): # NOQA """ Test that substituting different StorageClass parameters is reflected in the resulting PersistentVolumeClaim. Fixtures are torn down here in reverse order that they are specified as a parameter. Take caution when reordering test fixtures. """ # Prepare pod and volume specs. pod_name = 'provisioner-params-test' volume_size = 2 * Gi pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [ create_pvc_spec(pvc['metadata']['name']) ] pvc['spec']['resources']['requests']['storage'] = \ size_to_string(volume_size) pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME storage_class['parameters'] = { 'numberOfReplicas': '2', 'staleReplicaTimeout': '20' } create_storage(core_api, storage_class, pvc) create_and_wait_pod(core_api, pod) pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name']) # Confirm that the volume has all the correct parameters we gave it. volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == pvc_volume_name assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == \ int(storage_class['parameters']['numberOfReplicas']) assert volumes[0]["state"] == "attached"
def test_provisioner_tags(client, core_api, node_default_tags, storage_class, pvc, pod): # NOQA """ Test that a StorageClass can properly provision a volume with requested Tags. """ # Prepare pod and volume specs. pod_name = 'provisioner-tags-test' tag_spec = { "disk": ["ssd", "nvme"], "expected": 1, "node": ["storage", "main"] } pod['metadata']['name'] = pod_name pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])] pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME storage_class['parameters']['diskSelector'] = 'ssd,nvme' storage_class['parameters']['nodeSelector'] = 'storage,main' volume_size = DEFAULT_VOLUME_SIZE * Gi create_storage(core_api, storage_class, pvc) create_and_wait_pod(core_api, pod) pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name']) # Confirm that the volume has all the correct parameters we gave it. volumes = client.list_volume() assert len(volumes) == 1 assert volumes[0]["name"] == pvc_volume_name assert volumes[0]["size"] == str(volume_size) assert volumes[0]["numberOfReplicas"] == \ int(storage_class['parameters']['numberOfReplicas']) assert volumes[0]["state"] == "attached" check_volume_replicas(volumes[0], tag_spec, node_default_tags)
def test_recurring_job(set_random_backupstore, client, volume_name): # NOQA """ Test recurring job 1. Setup a random backupstore 2. Create a volume. 3. Create two jobs 1 job 1: snapshot every one minute, retain 2 1 job 2: backup every two minutes, retain 1 4. Attach the volume. Wait until the 10th second since the beginning of an even minute 5. Write some data. Sleep 2.5 minutes. Write some data. Sleep 2.5 minutes 6. Verify we have 4 snapshots total 1. 2 snapshots, 1 backup, 1 volume-head 7. Update jobs to replace the backup job 1. New backup job run every one minute, retain 2 8. Write some data. Sleep 2.5 minutes. Write some data. Sleep 2.5 minutes 9. We should have 6 snapshots 1. 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head 10. Make sure there are exactly 4 completed backups. 1. old backup job completed 2 backups 2. new backup job completed 2 backups 11. Make sure we have no backup in progress """ ''' The timeline looks like this: 0 1 2 3 4 5 6 7 8 9 10 (minute) |W | | W | | |W | | W | | | (write data) | S | S | | S | S | | (job_snap) | | B | B | | | | | | (job_backup1) | | | | | | B | B | | (job_backup2) ''' host_id = get_self_host_id() volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) jobs = create_jobs1() volume.recurringUpdate(jobs=jobs) volume = volume.attach(hostId=host_id) volume = wait_for_volume_healthy(client, volume_name) # wait until the beginning of an even minute wait_until_begin_of_an_even_minute() # wait until the 20th second of an even minute # make sure that snapshot job happens before the backup job time.sleep(20) write_volume_random_data(volume) time.sleep(120) # 2 minutes write_volume_random_data(volume) time.sleep(120) # 2 minutes check_jobs1_result(volume) job_backup2 = { "name": "backup2", "cron": "* * * * *", "task": "backup", "retain": 2 } volume.recurringUpdate(jobs=[jobs[0], job_backup2]) write_volume_random_data(volume) time.sleep(120) # 2 minutes write_volume_random_data(volume) time.sleep(120) # 2 minutes snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot.removed is False: count += 1 # 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head assert count == 6 complete_backup_number = 0 in_progress_backup_number = 0 volume = client.by_id_volume(volume_name) for b in volume.backupStatus: assert b.error == "" if b.state == "complete": complete_backup_number += 1 elif b.state == "in_progress": in_progress_backup_number += 1 # 2 completed backups from job_backup # 2 completed backups from job_backup2 assert complete_backup_number == 4 assert in_progress_backup_number == 0 volume = volume.detach(hostId="") common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_upgrade( upgrade_longhorn_manager_repo_url, upgrade_longhorn_manager_repo_branch, upgrade_longhorn_manager_image, upgrade_longhorn_engine_image, upgrade_longhorn_instance_manager_image, upgrade_longhorn_share_manager_image, upgrade_longhorn_backing_image_manager_image, client, core_api, volume_name, csi_pv, # NOQA pvc, pod_make, statefulset, storage_class): # NOQA """ Test Longhorn upgrade TODO The test will cover both volume has revision counter enabled and disabled cases. Prerequisite: - Disable Auto Salvage Setting 1. Find the upgrade image tag 2. Create a volume, generate and write data into the volume. 1. Create a volume with revision counter enabled case. 2. Create a volume with revision counter disabled case. 3. Create a Pod using a volume, generate and write data 4. Create a StatefulSet with 2 replicas, generate and write data to their volumes 5. Keep all volumes attached 6. Upgrade Longhorn system. 7. Check Pod and StatefulSet didn't restart after upgrade 8. Check All volumes data 9. Write data to StatefulSet pods, and Attached volume 10. Check data written to StatefulSet pods, and attached volume. 11. Detach the volume, and Delete Pod, and StatefulSet to detach theirvolumes 12. Upgrade all volumes engine images. 13. Attach the volume, and recreate Pod, and StatefulSet 14. Check All volumes data """ longhorn_manager_repo = upgrade_longhorn_manager_repo_url longhorn_manager_branch = upgrade_longhorn_manager_repo_branch longhorn_manager_image = upgrade_longhorn_manager_image longhorn_engine_image = upgrade_longhorn_engine_image longhorn_instance_manager_image = upgrade_longhorn_instance_manager_image longhorn_share_manager_image = upgrade_longhorn_share_manager_image longhorn_backing_image_manager_image = \ upgrade_longhorn_backing_image_manager_image host_id = get_self_host_id() pod_data_path = "/data/test" pod_volume_name = generate_volume_name() auto_salvage_setting = client.by_id_setting(SETTING_AUTO_SALVAGE) setting = client.update(auto_salvage_setting, value="false") assert setting.name == SETTING_AUTO_SALVAGE assert setting.value == "false" # Create Volume attached to a node. volume1 = create_and_check_volume(client, volume_name, size=SIZE) volume1.attach(hostId=host_id) volume1 = wait_for_volume_healthy(client, volume_name) volume1_data = write_volume_random_data(volume1) # Create Volume used by Pod pod_name, pv_name, pvc_name, pod_md5sum = \ prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc, pod_make, pod_volume_name, data_path=pod_data_path, add_liveness_probe=False) # Create multiple volumes used by StatefulSet statefulset_name = 'statefulset-upgrade-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) statefulset_pod_info = get_statefulset_pod_info(core_api, statefulset) for sspod_info in statefulset_pod_info: sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, sspod_info['pod_name'], sspod_info['data']) # upgrade Longhorn assert longhorn_upgrade(longhorn_manager_repo, longhorn_manager_branch, longhorn_manager_image, longhorn_engine_image, longhorn_instance_manager_image, longhorn_share_manager_image, longhorn_backing_image_manager_image) client = get_longhorn_api_client() # wait for 1 minute before checking pod restarts time.sleep(60) pod = core_api.read_namespaced_pod(name=pod_name, namespace='default') assert pod.status.container_statuses[0].restart_count == 0 for sspod_info in statefulset_pod_info: sspod = core_api.read_namespaced_pod(name=sspod_info['pod_name'], namespace='default') assert \ sspod.status.container_statuses[0].restart_count == 0 for sspod_info in statefulset_pod_info: resp = read_volume_data(core_api, sspod_info['pod_name']) assert resp == sspod_info['data'] res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path) assert res_pod_md5sum == pod_md5sum check_volume_data(volume1, volume1_data) for sspod_info in statefulset_pod_info: sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE) write_pod_volume_data(core_api, sspod_info['pod_name'], sspod_info['data']) for sspod_info in statefulset_pod_info: resp = read_volume_data(core_api, sspod_info['pod_name']) assert resp == sspod_info['data'] volume1 = client.by_id_volume(volume_name) volume1_data = write_volume_random_data(volume1) check_volume_data(volume1, volume1_data) statefulset['spec']['replicas'] = replicas = 0 apps_api = get_apps_api_client() apps_api.patch_namespaced_stateful_set( name=statefulset_name, namespace='default', body={'spec': { 'replicas': replicas }}) delete_and_wait_pod(core_api, pod_name) volume = client.by_id_volume(volume_name) volume.detach(hostId="") volumes = client.list_volume() for v in volumes: wait_for_volume_detached(client, v.name) engineimages = client.list_engine_image() for ei in engineimages: if ei.image == longhorn_engine_image: new_ei = ei volumes = client.list_volume() for v in volumes: volume = client.by_id_volume(v.name) volume.engineUpgrade(image=new_ei.image) statefulset['spec']['replicas'] = replicas = 2 apps_api = get_apps_api_client() apps_api.patch_namespaced_stateful_set( name=statefulset_name, namespace='default', body={'spec': { 'replicas': replicas }}) wait_statefulset(statefulset) pod = pod_make(name=pod_name) pod['spec']['volumes'] = [create_pvc_spec(pvc_name)] create_and_wait_pod(core_api, pod) volume1 = client.by_id_volume(volume_name) volume1.attach(hostId=host_id) volume1 = wait_for_volume_healthy(client, volume_name) for sspod_info in statefulset_pod_info: resp = read_volume_data(core_api, sspod_info['pod_name']) assert resp == sspod_info['data'] res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path) assert res_pod_md5sum == pod_md5sum check_volume_data(volume1, volume1_data)
def test_statefulset_scaling(client, core_api, storage_class, statefulset): # NOQA """ Test that scaling up a StatefulSet successfully provisions new volumes. 1. Create a StatefulSet with VolumeClaimTemplate and Longhorn. 2. Wait for pods to run. 3. Verify the properities of volumes. 4. Scale the StatefulSet to 3 replicas 5. Wait for the new pod to become ready. 6. Verify the new volume properties. """ statefulset_name = 'statefulset-scaling-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) pod_info = get_statefulset_pod_info(core_api, statefulset) volumes = client.list_volume() assert len(volumes) == statefulset['spec']['replicas'] for v in volumes: found = False for pod in pod_info: if v.name == pod['pv_name']: found = True break assert found pod_info.remove(pod) assert v.size == str(DEFAULT_VOLUME_SIZE * Gi) assert v.numberOfReplicas == \ int(storage_class['parameters']['numberOfReplicas']) assert v.state == 'attached' assert len(pod_info) == 0 statefulset['spec']['replicas'] = replicas = 3 apps_api = get_apps_api_client() apps_api.patch_namespaced_stateful_set( name=statefulset_name, namespace='default', body={'spec': { 'replicas': replicas }}) for i in range(DEFAULT_POD_TIMEOUT): s_set = apps_api.read_namespaced_stateful_set(name=statefulset_name, namespace='default') if s_set.status.ready_replicas == replicas: break time.sleep(DEFAULT_POD_INTERVAL) assert s_set.status.ready_replicas == replicas pod_info = get_statefulset_pod_info(core_api, statefulset) volumes = client.list_volume() assert len(volumes) == replicas for v in volumes: found = False for pod in pod_info: if v.name == pod['pv_name']: found = True break assert found pod_info.remove(pod) assert v.size == str(DEFAULT_VOLUME_SIZE * Gi) assert v.numberOfReplicas == \ int(storage_class['parameters']['numberOfReplicas']) assert v.state == 'attached' assert len(pod_info) == 0
def test_ha_simple_recovery(client, volume_name): # NOQA # get a random client volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) assert volume["name"] == volume_name assert volume["size"] == SIZE assert volume["numberOfReplicas"] == 2 assert volume["state"] == "detached" assert volume["created"] != "" host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) volume = client.by_id_volume(volume_name) assert volume["endpoint"] == DEV_PATH + volume_name assert len(volume["replicas"]) == 2 replica0 = volume["replicas"][0] assert replica0["name"] != "" replica1 = volume["replicas"][1] assert replica1["name"] != "" data = write_random_data(volume["endpoint"]) volume = volume.replicaRemove(name=replica0["name"]) # wait until we saw a replica starts rebuilding new_replica_found = False for i in range(RETRY_COUNTS): v = client.by_id_volume(volume_name) for r in v["replicas"]: if r["name"] != replica0["name"] and \ r["name"] != replica1["name"]: new_replica_found = True break if new_replica_found: break time.sleep(RETRY_ITERVAL) assert new_replica_found volume = common.wait_for_volume_healthy(client, volume_name) volume = client.by_id_volume(volume_name) assert volume["state"] == common.VOLUME_STATE_ATTACHED assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY assert len(volume["replicas"]) >= 2 found = False for replica in volume["replicas"]: if replica["name"] == replica1["name"]: found = True break assert found check_data(volume['endpoint'], data) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_recurring_job(clients, volume_name): # NOQA """ Test recurring job 1. Setup a random backupstore 2. Create a volume. 3. Create two jobs 1 job 1: snapshot every one minute, retain 2 1 job 2: backup every two minutes, retain 1 4. Attach the volume. 5. Sleep for 5 minutes 6. Verify we have 4 snapshots total 1. 2 snapshots, 1 backup, 1 volume-head 7. Update jobs to replace the backup job 1. New backup job run every one minute, retain 2 8. Sleep for 5 minutes. 9. We should have 6 snapshots 1. 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head 10. Make sure we have no more than 5 backups. 1. old backup job may have at most 1 backups 2. new backup job may have at most 3 backups 3. FIXME: Seems we should have at most 4 backups? 11. Make sure we have no more than 2 backups in progress 1. FIXME: Seems we should have at most 1 from the new job? """ for host_id, client in iter(clients.items()): # NOQA break set_random_backupstore(client) volume = client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2) volume = common.wait_for_volume_detached(client, volume_name) jobs = create_jobs1() volume.recurringUpdate(jobs=jobs) volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) # 5 minutes time.sleep(300) check_jobs1_result(volume) job_backup2 = { "name": "backup2", "cron": "* * * * *", "task": "backup", "retain": 2 } volume.recurringUpdate(jobs=[jobs[0], job_backup2]) # 5 minutes time.sleep(300) snapshots = volume.snapshotList() count = 0 for snapshot in snapshots: if snapshot.removed is False: count += 1 # 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head assert count == 6 complete_backup_number = 0 in_progress_backup_number = 0 volume = client.by_id_volume(volume_name) for b in volume.backupStatus: assert b.error == "" if b.state == "complete": complete_backup_number += 1 elif b.state == "in_progress": in_progress_backup_number += 1 assert complete_backup_number <= MAX_BACKUP_STATUS_SIZE # 1 from job_backup, 1 from job_backup2 assert in_progress_backup_number <= 2 volume = volume.detach() common.wait_for_volume_detached(client, volume_name) client.delete(volume) wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA volume = client.create_volume(name=volume_name, size=size, numberOfReplicas=2, baseImage=base_image) volume = common.wait_for_volume_detached(client, volume_name) host_id = get_self_host_id() volume = volume.attach(hostId=host_id) volume = common.wait_for_volume_healthy(client, volume_name) setting = client.by_id_setting(common.SETTING_BACKUP_TARGET) # test backupTarget for multiple settings backupstores = common.get_backupstore_url() for backupstore in backupstores: if common.is_backupTarget_s3(backupstore): backupsettings = backupstore.split("$") setting = client.update(setting, value=backupsettings[0]) assert setting["value"] == backupsettings[0] credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value=backupsettings[1]) assert credential["value"] == backupsettings[1] else: setting = client.update(setting, value=backupstore) assert setting["value"] == backupstore credential = client.by_id_setting( common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) credential = client.update(credential, value="") assert credential["value"] == "" data = write_volume_random_data(volume) snap2 = volume.snapshotCreate() volume.snapshotCreate() volume.snapshotBackup(name=snap2["name"]) _, b = common.find_backup(client, volume_name, snap2["name"]) res_name = common.generate_volume_name() res_volume = client.create_volume(name=res_name, size=size, numberOfReplicas=2, fromBackup=b["url"]) res_volume = common.wait_for_volume_detached(client, res_name) res_volume = res_volume.attach(hostId=host_id) res_volume = common.wait_for_volume_healthy(client, res_name) check_volume_data(res_volume, data) snapshots = res_volume.snapshotList() # only the backup snapshot + volume-head assert len(snapshots) == 2 backup_snapshot = "" for snap in snapshots: if snap["name"] != "volume-head": backup_snapshot = snap["name"] assert backup_snapshot != "" res_volume.snapshotCreate() snapshots = res_volume.snapshotList() assert len(snapshots) == 3 res_volume.snapshotDelete(name=backup_snapshot) res_volume.snapshotPurge() snapshots = res_volume.snapshotList() assert len(snapshots) == 2 ha_rebuild_replica_test(client, res_name) res_volume = res_volume.detach() res_volume = common.wait_for_volume_detached(client, res_name) client.delete(res_volume) common.wait_for_volume_delete(client, res_name) volume = volume.detach() volume = common.wait_for_volume_detached(client, volume_name) client.delete(volume) common.wait_for_volume_delete(client, volume_name) volumes = client.list_volume() assert len(volumes) == 0
def test_statefulset_scaling(client, core_api, storage_class, statefulset): # NOQA """ Test that scaling up a StatefulSet successfully provisions new volumes. """ statefulset_name = 'statefulset-scaling-test' update_statefulset_manifests(statefulset, storage_class, statefulset_name) create_storage_class(storage_class) create_and_wait_statefulset(statefulset) pod_info = get_statefulset_pod_info(core_api, statefulset) volumes = client.list_volume() assert len(volumes) == statefulset['spec']['replicas'] for v in volumes: found = False for pod in pod_info: if v['name'] == pod['pv_name']: found = True break assert found pod_info.remove(pod) assert v['size'] == str(DEFAULT_VOLUME_SIZE * Gi) assert v['numberOfReplicas'] == \ int(storage_class['parameters']['numberOfReplicas']) assert v['state'] == 'attached' assert len(pod_info) == 0 statefulset['spec']['replicas'] = replicas = 3 apps_api = get_apps_api_client() apps_api.patch_namespaced_stateful_set( name=statefulset_name, namespace='default', body={ 'spec': { 'replicas': replicas } }) for i in range(DEFAULT_POD_TIMEOUT): s_set = apps_api.read_namespaced_stateful_set( name=statefulset_name, namespace='default') if s_set.status.ready_replicas == replicas: break time.sleep(DEFAULT_POD_INTERVAL) assert s_set.status.ready_replicas == replicas pod_info = get_statefulset_pod_info(core_api, statefulset) volumes = client.list_volume() assert len(volumes) == replicas for v in volumes: found = False for pod in pod_info: if v['name'] == pod['pv_name']: found = True break assert found pod_info.remove(pod) assert v['size'] == str(DEFAULT_VOLUME_SIZE * Gi) assert v['numberOfReplicas'] == \ int(storage_class['parameters']['numberOfReplicas']) assert v['state'] == 'attached' assert len(pod_info) == 0