def backup_test(client): # NOQA
volume = client.by_id_volume(VOLUME_NAME)
setting = client.by_id_setting("backupTarget")
setting = client.update(setting, value=common.get_backupstore_url())
assert setting["value"] == common.get_backupstore_url()
volume.snapshotCreate()
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
found = False
for i in range(100):
bvs = client.list_backupVolume()
for bv in bvs:
if bv["name"] == VOLUME_NAME:
found = True
break
if found:
break
time.sleep(1)
assert found
found = False
for i in range(20):
backups = bv.backupList()
for b in backups:
if b["snapshotName"] == snap2["name"]:
found = True
break
if found:
break
time.sleep(1)
assert found
new_b = bv.backupGet(name=b["name"])
assert new_b["name"] == b["name"]
assert new_b["url"] == b["url"]
assert new_b["snapshotName"] == b["snapshotName"]
assert new_b["snapshotCreated"] == b["snapshotCreated"]
assert new_b["created"] == b["created"]
assert new_b["volumeName"] == b["volumeName"]
assert new_b["volumeSize"] == b["volumeSize"]
assert new_b["volumeCreated"] == b["volumeCreated"]
bv.backupDelete(name=b["name"])
backups = bv.backupList()
found = False
for b in backups:
if b["snapshotName"] == snap2["name"]:
found = True
break
assert not found
def csi_backup_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA
pod_name = 'csi-backup-test'
create_and_wait_csi_pod(pod_name, client, core_api, csi_pv, pvc, pod_make,
base_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
i = 1
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting.value == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential.value == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting.value == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential.value == ""
backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name,
base_image, test_data, i)
i += 1
def test_bgtasks(clients): # NOQA
for host_id, client in clients.iteritems():
break
setting = client.by_id_setting("backupTarget")
setting = client.update(setting, value=common.get_backupstore_url())
volume = client.create_volume(name=VOLUME_NAME,
size=SIZE,
numberOfReplicas=2)
volume = volume.attach(hostId=host_id)
num_tasks = 4
snaps = map(lambda _: volume.snapshotCreate(), range(num_tasks))
for snap in snaps:
volume.snapshotBackup(name=snap["name"])
queue = volume.bgTaskQueue()
q_len = len(queue)
assert len(queue) == num_tasks
assert queue[0]["num"] == 1
assert queue[0]["finished"] == "" # running: finished is not set
while q_len > 1:
queue = volume.bgTaskQueue()
if len(queue) < q_len:
q_len = len(queue)
assert queue[0]["finished"] != "" # done: finished is set
assert queue[0]["err"] is None
assert queue[0]["num"] == 1 + num_tasks - q_len
else:
time.sleep(1)
def csi_backup_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA
pod_name = 'csi-backup-test'
create_and_wait_csi_pod(pod_name, client, core_api, csi_pv, pvc, pod_make,
base_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
i = 1
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name,
base_image, test_data, i)
i += 1
def test_restore_inc(clients, core_api, volume_name, pod): # NOQA
for _, client in clients.iteritems():
break
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
restore_inc_test(client, core_api, volume_name, pod)
def backup_test(clients, volume_name, size, base_image=""): # NOQA
for host_id, client in clients.iteritems():
break
volume = create_and_check_volume(client, volume_name, 2, size, base_image)
lht_hostId = get_self_host_id()
volume = volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
backupstore_test(client, lht_hostId, volume_name, size)
cleanup_volume(client, volume)
def set_backupstore_nfs(client):
backupstores = get_backupstore_url()
poll_interval = get_backupstore_poll_interval()
for backupstore in backupstores:
if is_backupTarget_nfs(backupstore):
set_backupstore_url(client, backupstore)
set_backupstore_credential_secret(client, "")
set_backupstore_poll_interval(client, poll_interval)
break
def set_backupstore_s3(client):
backupstores = get_backupstore_url()
poll_interval = get_backupstore_poll_interval()
for backupstore in backupstores:
if is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
set_backupstore_url(client, backupsettings[0])
set_backupstore_credential_secret(client, backupsettings[1])
set_backupstore_poll_interval(client, poll_interval)
break
def backup_test(clients, volume_name, size, base_image=""): # NOQA
for host_id, client in clients.iteritems():
break
volume = client.create_volume(name=volume_name,
size=size,
numberOfReplicas=2,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == size
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["baseImage"] == base_image
lht_hostId = get_self_host_id()
volume = volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
backupstore_test(client, lht_hostId, volume_name, size)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
volume = wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def set_backupstore_nfs(client):
backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
backupstores = get_backupstore_url()
for backupstore in backupstores:
if is_backupTarget_nfs(backupstore):
backup_target_setting = client.update(backup_target_setting,
value=backupstore)
assert backup_target_setting.value == backupstore
backup_target_credential_setting = client.by_id_setting(
SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
backup_target_credential_setting = \
client.update(backup_target_credential_setting, value="")
assert backup_target_credential_setting.value == ""
break
def backup_labels_test(clients,
random_labels,
volume_name,
size=SIZE,
base_image=""): # NOQA
for _, client in clients.iteritems():
break
host_id = get_self_host_id()
volume = create_and_check_volume(client, volume_name, 2, size, base_image)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
bv, b, _, _ = create_backup(client, volume_name, labels=random_labels)
# If we're running the test with a BaseImage, check that this Label is
# set properly.
backup = bv.backupGet(name=b["name"])
if base_image:
assert backup["labels"].get(common.BASE_IMAGE_LABEL) == base_image
# One extra Label from the BaseImage being set.
assert len(backup["labels"]) == len(random_labels) + 1
else:
assert len(backup["labels"]) == len(random_labels)
cleanup_volume(client, volume)
def set_backupstore_s3(client):
backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
backupstores = get_backupstore_url()
for backupstore in backupstores:
if is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
backup_target_setting = client.update(backup_target_setting,
value=backupsettings[0])
assert backup_target_setting.value == backupsettings[0]
backup_target_credential_setting = client.by_id_setting(
SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
backup_target_credential_setting = \
client.update(backup_target_credential_setting,
value=backupsettings[1])
assert backup_target_credential_setting.value == backupsettings[1]
break
def backup_test(client, host_id): # NOQA
volume = client.by_id_volume(VOLUME_NAME)
setting = client.by_id_setting("backupTarget")
setting = client.update(setting, value=common.get_backupstore_url())
assert setting["value"] == common.get_backupstore_url()
volume.snapshotCreate()
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
jobs = volume.jobList()
found = False
for job in jobs:
if job["state"] == "ongoing" and job["jobType"] == "snapshot-backup":
found = True
break
assert found
found = False
for i in range(100):
bvs = client.list_backupVolume()
for bv in bvs:
if bv["name"] == VOLUME_NAME:
found = True
break
if found:
break
time.sleep(1)
assert found
found = False
for i in range(20):
backups = bv.backupList()
for b in backups:
if b["snapshotName"] == snap2["name"]:
found = True
break
if found:
break
time.sleep(1)
assert found
new_b = bv.backupGet(name=b["name"])
assert new_b["name"] == b["name"]
assert new_b["url"] == b["url"]
assert new_b["snapshotName"] == b["snapshotName"]
assert new_b["snapshotCreated"] == b["snapshotCreated"]
assert new_b["created"] == b["created"]
assert new_b["volumeName"] == b["volumeName"]
assert new_b["volumeSize"] == b["volumeSize"]
assert new_b["volumeCreated"] == b["volumeCreated"]
# test restore
volume = client.create_volume(name=VOLUME_RESTORE_NAME,
size=SIZE,
numberOfReplicas=2,
fromBackup=b["url"])
volume = wait_for_volume_state(client, VOLUME_RESTORE_NAME, "detached")
assert volume["name"] == VOLUME_RESTORE_NAME
assert volume["size"] == SIZE
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
volume = volume.attach(hostId=host_id)
volume = wait_for_volume_state(client, VOLUME_RESTORE_NAME, "healthy")
volume = volume.detach()
volume = wait_for_volume_state(client, VOLUME_RESTORE_NAME, "detached")
client.delete(volume)
volume = wait_for_volume_delete(client, VOLUME_RESTORE_NAME)
bv.backupDelete(name=b["name"])
backups = bv.backupList()
found = False
for b in backups:
if b["snapshotName"] == snap2["name"]:
found = True
break
assert not found
def ha_backup_deletion_recovery_test(client,
volume_name,
size,
base_image=""): # NOQA
volume = client.create_volume(name=volume_name,
size=size,
numberOfReplicas=2,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
data = write_volume_random_data(volume)
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
_, b = common.find_backup(client, volume_name, snap2["name"])
res_name = common.generate_volume_name()
res_volume = client.create_volume(name=res_name,
size=size,
numberOfReplicas=2,
fromBackup=b["url"])
res_volume = common.wait_for_volume_restoration_completed(
client, res_name)
res_volume = common.wait_for_volume_detached(client, res_name)
res_volume = res_volume.attach(hostId=host_id)
res_volume = common.wait_for_volume_healthy(client, res_name)
check_volume_data(res_volume, data)
snapshots = res_volume.snapshotList()
# only the backup snapshot + volume-head
assert len(snapshots) == 2
backup_snapshot = ""
for snap in snapshots:
if snap["name"] != "volume-head":
backup_snapshot = snap["name"]
assert backup_snapshot != ""
res_volume.snapshotCreate()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 3
res_volume.snapshotDelete(name=backup_snapshot)
res_volume.snapshotPurge()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 2
ha_rebuild_replica_test(client, res_name)
res_volume = res_volume.detach()
res_volume = common.wait_for_volume_detached(client, res_name)
client.delete(res_volume)
common.wait_for_volume_delete(client, res_name)
cleanup_volume(client, volume)
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA
volume = client.create_volume(name=volume_name, size=size,
numberOfReplicas=2, baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
data = write_volume_random_data(volume)
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
_, b = common.find_backup(client, volume_name, snap2["name"])
res_name = common.generate_volume_name()
res_volume = client.create_volume(name=res_name, size=size,
numberOfReplicas=2,
fromBackup=b["url"])
res_volume = common.wait_for_volume_detached(client, res_name)
res_volume = res_volume.attach(hostId=host_id)
res_volume = common.wait_for_volume_healthy(client, res_name)
check_volume_data(res_volume, data)
snapshots = res_volume.snapshotList()
# only the backup snapshot + volume-head
assert len(snapshots) == 2
backup_snapshot = ""
for snap in snapshots:
if snap["name"] != "volume-head":
backup_snapshot = snap["name"]
assert backup_snapshot != ""
res_volume.snapshotCreate()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 3
res_volume.snapshotDelete(name=backup_snapshot)
res_volume.snapshotPurge()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 2
ha_rebuild_replica_test(client, res_name)
res_volume = res_volume.detach()
res_volume = common.wait_for_volume_detached(client, res_name)
client.delete(res_volume)
common.wait_for_volume_delete(client, res_name)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_listing_backup_volume(clients, base_image=""): # NOQA
for host_id, client in clients.iteritems():
break
lht_hostId = get_self_host_id()
# create 3 volumes.
volume1_name = generate_volume_name()
volume2_name = generate_volume_name()
volume3_name = generate_volume_name()
volume1 = create_and_check_volume(client, volume1_name)
volume2 = create_and_check_volume(client, volume2_name)
volume3 = create_and_check_volume(client, volume3_name)
volume1.attach(hostId=lht_hostId)
volume1 = common.wait_for_volume_healthy(client, volume1_name)
volume2.attach(hostId=lht_hostId)
volume2 = common.wait_for_volume_healthy(client, volume2_name)
volume3.attach(hostId=lht_hostId)
volume3 = common.wait_for_volume_healthy(client, volume3_name)
# we only test NFS here.
# Since it is difficult to directly remove volume.cfg from s3 buckets
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_nfs(backupstore):
updated = False
for i in range(RETRY_COMMAND_COUNT):
nfs_url = backupstore.strip("nfs://")
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
if "nfs" in setting["value"]:
updated = True
break
assert updated
_, _, snap1, _ = create_backup(client, volume1_name)
_, _, snap2, _ = create_backup(client, volume2_name)
_, _, snap3, _ = create_backup(client, volume3_name)
# invalidate backup volume 1 by renaming volume.cfg to volume.cfg.tmp
cmd = ["mkdir", "-p", "/mnt/nfs"]
subprocess.check_output(cmd)
cmd = ["mount", "-t", "nfs4", nfs_url, "/mnt/nfs"]
subprocess.check_output(cmd)
cmd = ["find", "/mnt/nfs", "-type", "d", "-name", volume1_name]
volume1_backup_volume_path = subprocess.check_output(cmd).strip()
cmd = ["find", volume1_backup_volume_path, "-name", "volume.cfg"]
volume1_backup_volume_cfg_path = subprocess.check_output(cmd).strip()
cmd = [
"mv", volume1_backup_volume_cfg_path,
volume1_backup_volume_cfg_path + ".tmp"
]
subprocess.check_output(cmd)
subprocess.check_output(["sync"])
found1 = found2 = found3 = False
for i in range(RETRY_COUNTS):
bvs = client.list_backupVolume()
for bv in bvs:
if bv["name"] == volume1_name:
if "error" in bv.messages:
assert "volume.cfg" in bv.messages["error"].lower()
found1 = True
elif bv["name"] == volume2_name:
assert not bv.messages
found2 = True
elif bv["name"] == volume3_name:
assert not bv.messages
found3 = True
if found1 & found2 & found3:
break
time.sleep(RETRY_INTERVAL)
assert found1 & found2 & found3
cmd = [
"mv", volume1_backup_volume_cfg_path + ".tmp",
volume1_backup_volume_cfg_path
]
subprocess.check_output(cmd)
subprocess.check_output(["sync"])
found = False
for i in range(RETRY_COMMAND_COUNT):
try:
bv1, b1 = common.find_backup(client, volume1_name, snap1["name"])
found = True
break
except Exception:
time.sleep(1)
assert found
bv1.backupDelete(name=b1["name"])
for i in range(RETRY_COMMAND_COUNT):
found = False
backups1 = bv1.backupList()
for b in backups1:
if b["snapshotName"] == snap1["name"]:
found = True
break
assert not found
bv2, b2 = common.find_backup(client, volume2_name, snap2["name"])
bv2.backupDelete(name=b2["name"])
for i in range(RETRY_COMMAND_COUNT):
found = False
backups2 = bv2.backupList()
for b in backups2:
if b["snapshotName"] == snap2["name"]:
found = True
break
assert not found
bv3, b3 = common.find_backup(client, volume3_name, snap3["name"])
bv3.backupDelete(name=b3["name"])
for i in range(RETRY_COMMAND_COUNT):
found = False
backups3 = bv3.backupList()
for b in backups3:
if b["snapshotName"] == snap3["name"]:
found = True
break
assert not found
volume1.detach()
volume1 = common.wait_for_volume_detached(client, volume1_name)
client.delete(volume1)
wait_for_volume_delete(client, volume1_name)
volume2.detach()
volume2 = common.wait_for_volume_detached(client, volume2_name)
client.delete(volume2)
wait_for_volume_delete(client, volume2_name)
volume3.detach()
volume3 = common.wait_for_volume_detached(client, volume3_name)
client.delete(volume3)
wait_for_volume_delete(client, volume3_name)
volumes = client.list_volume()
assert len(volumes) == 0
