def ha_salvage_test(client, volume_name, base_image=""): # NOQA
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2, baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == SIZE
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["created"] != ""
assert volume["baseImage"] == base_image
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 2
replica0_name = volume["replicas"][0]["name"]
replica1_name = volume["replicas"][1]["name"]
data = write_volume_random_data(volume)
common.k8s_delete_replica_pods_for_volume(volume_name)
volume = common.wait_for_volume_faulted(client, volume_name)
assert len(volume["replicas"]) == 2
assert volume["replicas"][0]["failedAt"] != ""
assert volume["replicas"][1]["failedAt"] != ""
volume.salvage(names=[replica0_name, replica1_name])
volume = common.wait_for_volume_detached(client, volume_name)
assert len(volume["replicas"]) == 2
assert volume["replicas"][0]["failedAt"] == ""
assert volume["replicas"][1]["failedAt"] == ""
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
check_volume_data(volume, data)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def migration_confirm_test(clients, volume_name, base_image=""): # NOQA
client = get_random_client(clients)
hosts = clients.keys()
host1 = hosts[0]
host2 = hosts[1]
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=REPLICA_COUNT,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host1)
volume = common.wait_for_volume_healthy(client, volume_name)
volume = volume.migrationStart(nodeId=host2)
attached_nodes = get_volume_attached_nodes(volume)
assert host1 in attached_nodes
assert volume["migrationNodeID"] == host2
with pytest.raises(Exception) as e:
volume.migrationConfirm()
assert "migration is not ready" in str(e.value)
volume = common.wait_for_volume_migration_ready(client, volume_name)
volume = volume.migrationConfirm()
volume = common.wait_for_volume_migration_node(client,
volume_name,
host2)
assert volume["migrationNodeID"] == ""
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
def ha_simple_recovery_test(client, volume_name, size, base_image=""): # NOQA
volume = client.create_volume(name=volume_name, size=size,
numberOfReplicas=2, baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == size
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["created"] != ""
assert volume["baseImage"] == base_image
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
ha_rebuild_replica_test(client, volume_name)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def migration_rollback_test(clients, volume_name, base_image=""): # NOQA
client = get_random_client(clients)
hosts = clients.keys()
host1 = hosts[0]
host2 = hosts[1]
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=REPLICA_COUNT,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host1)
volume = common.wait_for_volume_healthy(client, volume_name)
volume = volume.migrationStart(nodeId=host2)
attached_nodes = get_volume_attached_nodes(volume)
assert host1 in attached_nodes
assert volume["migrationNodeID"] == host2
volume = common.wait_for_volume_migration_ready(client, volume_name)
volume = volume.migrationRollback()
volume = common.wait_for_volume_migration_node(client, volume_name, host1)
assert volume["migrationNodeID"] == ""
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
def test_ha_prohibit_deleting_last_replica(client, volume_name): # NOQA
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=1)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == SIZE
assert volume["numberOfReplicas"] == 1
assert volume["state"] == "detached"
assert volume["created"] != ""
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 1
replica0 = volume["replicas"][0]
with pytest.raises(Exception) as e:
volume.replicaRemove(name=replica0["name"])
assert "no other healthy replica available" in str(e.value)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA
for host_id, client in clients.iteritems(): # NOQA
break
# error when creating volume with duplicate jobs
with pytest.raises(Exception) as e:
client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1() + create_jobs1())
assert "duplicate job" in str(e.value)
client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2, recurringJobs=create_jobs1())
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_replica_scheduler_update_over_provisioning(client): # NOQA
nodes = client.list_node()
lht_hostId = get_self_host_id()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
over_provisioning_setting = client.by_id_setting(
SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE)
old_provisioning_setting = over_provisioning_setting["value"]
# set storage over provisioning percentage to 0
# to test all replica couldn't be scheduled
over_provisioning_setting = client.update(over_provisioning_setting,
value="0")
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE, numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
# set storage over provisioning percentage to 100
over_provisioning_setting = client.update(over_provisioning_setting,
value="100")
# check volume status
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
client.update(over_provisioning_setting,
value=old_provisioning_setting)
def test_hard_anti_affinity_detach(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity are still able to detach and
reattach to a node properly, even in degraded state.
1. Create a volume and attach to the current node
2. Generate and write `data` to the volume.
3. Set `soft anti-affinity` to false
4. Disable current node's scheduling.
5. Remove the replica on the current node
1. Verify volume will be in degraded state.
2. Verify volume reports condition `scheduled == false`
6. Detach the volume.
7. Verify that volume only have 2 replicas
1. Unhealthy replica will be removed upon detach.
8. Attach the volume again.
1. Verify volume will be in degraded state.
2. Verify volume reports condition `scheduled == false`
3. Verify only two of three replicas of volume are healthy.
4. Verify the remaining replica doesn't have `replica.HostID`, meaning
it's unscheduled
9. Check volume `data`
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica.name)
volume = wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
volume.detach(hostId="")
volume = wait_for_volume_detached(client, volume_name)
assert len(volume.replicas) == 2
volume.attach(hostId=host_id)
# Make sure we're still not getting another successful replica.
volume = wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
assert sum([
1 for replica in volume.replicas
if replica.running and replica.mode == "RW"
]) == 2
assert sum([1 for replica in volume.replicas if not replica.hostId]) == 1
assert len(volume.replicas) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_soft_anti_affinity_detach(client, volume_name): # NOQA
"""
Test that volumes with Soft Anti-Affinity can detach and reattach to a
node properly.
1. Create a volume and attach to the current node.
2. Generate and write `data` to the volume
3. Set `soft anti-affinity` to true
4. Disable current node's scheduling.
5. Remove the replica on the current node
6. Wait for the new replica to be rebuilt
7. Detach the volume.
8. Verify there are 3 replicas
9. Attach the volume again. Verify there are still 3 replicas
10. Verify the `data`.
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="true")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
replica_names = list(map(lambda replica: replica.name, volume.replicas))
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica.name)
wait_new_replica_ready(client, volume_name, replica_names)
volume = wait_for_volume_healthy(client, volume_name)
volume.detach()
volume = wait_for_volume_detached(client, volume_name)
assert len(volume.replicas) == 3
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def ha_simple_recovery_test(client, volume_name, size, base_image=""): # NOQA
volume = create_and_check_volume(client, volume_name, 2, size, base_image)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
ha_rebuild_replica_test(client, volume_name)
cleanup_volume(client, volume)
def test_hard_anti_affinity_scheduling(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity work as expected.
With Hard Anti-Affinity, scheduling on nodes with existing replicas should
be forbidden, resulting in "Degraded" state.
1. Create a volume and attach to the current node
2. Generate and write `data` to the volume.
3. Set `soft anti-affinity` to false
4. Disable current node's scheduling.
5. Remove the replica on the current node
1. Verify volume will be in degraded state.
2. Verify volume reports condition `scheduled == false`
3. Verify only two of three replicas of volume are healthy.
4. Verify the remaining replica doesn't have `replica.HostID`, meaning
it's unscheduled
6. Check volume `data`
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica.name)
# Instead of waiting for timeout and lengthening the tests a significant
# amount we can make sure the scheduling isn't working by making sure the
# volume becomes Degraded and reports a scheduling error.
wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
# While there are three replicas that should exist to meet the Volume's
# request, only two of those volumes should actually be Healthy.
volume = client.by_id_volume(volume_name)
assert sum([
1 for replica in volume.replicas
if replica.running and replica.mode == "RW"
]) == 2
# Confirm that the final volume is an unscheduled volume.
assert sum([1 for replica in volume.replicas if not replica.hostId]) == 1
# Three replicas in total should still exist.
assert len(volume.replicas) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def volume_iscsi_basic_test(clients, volume_name, base_image=""): # NOQA
# get a random client
for host_id, client in clients.iteritems():
break
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=3,
frontend="iscsi",
baseImage=base_image)
assert volume["name"] == volume_name
assert volume["size"] == SIZE
assert volume["numberOfReplicas"] == 3
assert volume["frontend"] == "iscsi"
assert volume["baseImage"] == base_image
volume = common.wait_for_volume_detached(client, volume_name)
assert len(volume["replicas"]) == 3
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 1
assert volumes[0]["name"] == volume["name"]
assert volumes[0]["size"] == volume["size"]
assert volumes[0]["numberOfReplicas"] == volume["numberOfReplicas"]
assert volumes[0]["state"] == volume["state"]
assert volumes[0]["created"] == volume["created"]
assert volumes[0]["frontend"] == "iscsi"
endpoint = get_volume_endpoint(volumes[0])
assert endpoint.startswith("iscsi://")
try:
dev = iscsi_login(endpoint)
volume_rw_test(dev)
finally:
iscsi_logout(endpoint)
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def revert_random_snapshot(client, core_api, volume_name, pod_manifest,
snapshots_md5sum): # NOQA
volume = client.by_id_volume(volume_name)
host_id = get_self_host_id()
pod_name = pod_manifest["metadata"]["name"]
# wait for volume healthy if rebuilding deleted replica
if len(volume.robustness) != VOLUME_ROBUSTNESS_HEALTHY:
wait_for_volume_healthy(client, volume_name)
snapshot = get_random_snapshot(snapshots_md5sum)
if snapshot is None:
print("skipped, no snapshot found", end=" ")
return
delete_and_wait_pod(core_api, pod_name)
wait_for_volume_detached(client, volume_name)
volume = client.by_id_volume(volume_name)
volume.attach(hostId=host_id, disableFrontend=True)
volume = wait_for_volume_healthy_no_frontend(client, volume_name)
volume.snapshotRevert(name=snapshot)
volume = client.by_id_volume(volume_name)
volume.detach()
wait_for_volume_detached(client, volume_name)
create_and_wait_pod(core_api, pod_manifest)
current_md5sum = read_data_md5sum(core_api, pod_name)
assert current_md5sum == snapshots_md5sum[snapshot].data_md5sum
def check_volume_data(client, volume_name, data): # NOQA
"""
Attaches the volume to the current node
then compares the volumes data
against the passed data.
"""
volume = client.by_id_volume(volume_name)
volume.attach(hostId=common.get_self_host_id())
volume = common.wait_for_volume_healthy(client, volume_name)
common.check_volume_data(volume, data)
volume.detach(hostId="")
volume = common.wait_for_volume_detached(client, volume_name)
def backupstore_test(client, host_id, volname, size):
volume = client.by_id_volume(volname)
volume.snapshotCreate()
data = write_volume_random_data(volume)
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
bv, b = common.find_backup(client, volname, snap2["name"])
new_b = bv.backupGet(name=b["name"])
assert new_b["name"] == b["name"]
assert new_b["url"] == b["url"]
assert new_b["snapshotName"] == b["snapshotName"]
assert new_b["snapshotCreated"] == b["snapshotCreated"]
assert new_b["created"] == b["created"]
assert new_b["volumeName"] == b["volumeName"]
assert new_b["volumeSize"] == b["volumeSize"]
assert new_b["volumeCreated"] == b["volumeCreated"]
# test restore
restoreName = generate_volume_name()
volume = client.create_volume(name=restoreName,
size=size,
numberOfReplicas=2,
fromBackup=b["url"])
volume = common.wait_for_volume_detached(client, restoreName)
assert volume["name"] == restoreName
assert volume["size"] == size
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, restoreName)
check_volume_data(volume, data)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, restoreName)
client.delete(volume)
volume = wait_for_volume_delete(client, restoreName)
bv.backupDelete(name=b["name"])
backups = bv.backupList()
found = False
for b in backups:
if b["snapshotName"] == snap2["name"]:
found = True
break
assert not found
def backing_image_basic_operation_test(client, volume_name, bi_name,
bi_url): # NOQA
"""
Test Backing Image APIs.
1. Create a backing image.
2. Create and attach a Volume with the backing image set.
3. Verify that the all disk states in the backing image are "downloaded".
4. Try to use the API to manually clean up one disk for the backing image
but get failed.
5. Try to use the API to directly delete the backing image
but get failed.
6. Delete the volume.
7. Use the API to manually clean up one disk for the backing image
8. Delete the backing image.
"""
volume = create_and_check_volume(client, volume_name, 3,
str(BACKING_IMAGE_EXT4_SIZE), bi_name)
lht_host_id = get_self_host_id()
volume.attach(hostId=lht_host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert volume.backingImage == bi_name
assert volume.size == str(BACKING_IMAGE_EXT4_SIZE)
random_disk_id = ""
backing_image = client.by_id_backing_image(bi_name)
assert backing_image.sourceType == BACKING_IMAGE_SOURCE_TYPE_DOWNLOAD
assert backing_image.parameters["url"] == bi_url
assert backing_image.currentChecksum != ""
assert not backing_image.deletionTimestamp
assert len(backing_image.diskFileStatusMap) == 3
for disk_id, status in iter(backing_image.diskFileStatusMap.items()):
assert status.state == "ready"
random_disk_id = disk_id
assert random_disk_id != ''
with pytest.raises(Exception):
backing_image.backingImageCleanup(disks=[random_disk_id])
with pytest.raises(Exception):
client.delete(backing_image)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
backing_image = client.by_id_backing_image(bi_name)
backing_image.backingImageCleanup(disks=[random_disk_id])
backing_image = wait_for_backing_image_disk_cleanup(
client, bi_name, random_disk_id)
client.delete(backing_image)
def backup_test(clients, volume_name, size, base_image=""): # NOQA
for host_id, client in clients.iteritems():
break
volume = client.create_volume(name=volume_name,
size=size,
numberOfReplicas=2,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == size
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["baseImage"] == base_image
lht_hostId = get_self_host_id()
volume = volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
backupstore_test(client, lht_hostId, volume_name, size)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
volume = wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_volume_update_replica_count(clients, volume_name): # NOQA
for host_id, client in clients.iteritems():
break
replica_count = 3
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=replica_count)
volume = common.wait_for_volume_detached(client, volume_name)
assert len(volume["replicas"]) == replica_count
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
replica_count = 5
volume = volume.updateReplicaCount(replicaCount=replica_count)
volume = common.wait_for_volume_degraded(client, volume_name)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == replica_count
old_replica_count = replica_count
replica_count = 2
volume = volume.updateReplicaCount(replicaCount=replica_count)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == old_replica_count
volume.replicaRemove(name=volume["replicas"][0]["name"])
volume.replicaRemove(name=volume["replicas"][1]["name"])
volume.replicaRemove(name=volume["replicas"][2]["name"])
volume = common.wait_for_volume_replica_count(client, volume_name,
replica_count)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == replica_count
client.delete(volume)
wait_for_volume_delete(client, volume_name)
def test_soft_anti_affinity_scheduling(client, volume_name): # NOQA
"""
Test that volumes with Soft Anti-Affinity work as expected.
With Soft Anti-Affinity, a new replica should still be scheduled on a node
with an existing replica, which will result in "Healthy" state but limited
redundancy.
1. Create a volume and attach to the current node
2. Generate and write `data` to the volume.
3. Set `soft anti-affinity` to true
4. Disable current node's scheduling.
5. Remove the replica on the current node
6. Wait for the volume to complete rebuild. Volume should have 3 replicas.
7. Verify `data`
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="true")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
replica_names = list(map(lambda replica: replica.name, volume.replicas))
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica.name)
wait_new_replica_ready(client, volume_name, replica_names)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def recurring_job_labels_test(client,
labels,
volume_name,
size=SIZE,
base_image=""): # NOQA
host_id = get_self_host_id()
client.create_volume(name=volume_name, size=size, numberOfReplicas=2)
volume = common.wait_for_volume_detached(client, volume_name)
# Simple Backup Job that runs every 2 minutes, retains 1.
jobs = [{
"name": RECURRING_JOB_NAME,
"cron": "*/2 * * * *",
"task": "backup",
"retain": 1,
"labels": labels
}]
volume.recurringUpdate(jobs=jobs)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot["removed"] is False:
count += 1
# 1 from Backup, 1 from Volume Head.
assert count == 2
# Verify the Labels on the actual Backup.
bv = client.by_id_backupVolume(volume_name)
backups = bv.backupList()
assert len(backups) == 1
b = bv.backupGet(name=backups[0]["name"])
for key, val in labels.iteritems():
assert b["labels"].get(key) == val
assert b["labels"].get(RECURRING_JOB_LABEL) == RECURRING_JOB_NAME
if base_image:
assert b["labels"].get(BASE_IMAGE_LABEL) == base_image
# One extra Label from the BaseImage being set.
assert len(b["labels"]) == len(labels) + 2
else:
# At least one extra Label from RecurringJob.
assert len(b["labels"]) == len(labels) + 1
cleanup_volume(client, volume)
def ha_salvage_test(client, volume_name, base_image=""): # NOQA
volume = create_and_check_volume(client,
volume_name,
2,
base_image=base_image)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 2
replica0_name = volume["replicas"][0]["name"]
replica1_name = volume["replicas"][1]["name"]
data = write_volume_random_data(volume)
common.k8s_delete_replica_pods_for_volume(volume_name)
volume = common.wait_for_volume_faulted(client, volume_name)
assert len(volume["replicas"]) == 2
assert volume["replicas"][0]["failedAt"] != ""
assert volume["replicas"][1]["failedAt"] != ""
volume.salvage(names=[replica0_name, replica1_name])
volume = common.wait_for_volume_detached(client, volume_name)
assert len(volume["replicas"]) == 2
assert volume["replicas"][0]["failedAt"] == ""
assert volume["replicas"][1]["failedAt"] == ""
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
check_volume_data(volume, data)
cleanup_volume(client, volume)
def test_ha_prohibit_deleting_last_replica(client, volume_name): # NOQA
volume = create_and_check_volume(client, volume_name, 1)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 1
replica0 = volume["replicas"][0]
with pytest.raises(Exception) as e:
volume.replicaRemove(name=replica0["name"])
assert "no other healthy replica available" in str(e.value)
cleanup_volume(client, volume)
def test_recurring_job_in_volume_creation(set_random_backupstore, client,
volume_name): # NOQA
"""
Test create volume with recurring jobs
1. Create volume with recurring jobs though Longhorn API
2. Verify the recurring jobs run correctly
"""
host_id = get_self_host_id()
# error when creating volume with duplicate jobs
with pytest.raises(Exception) as e:
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1() + create_jobs1())
assert "duplicate job" in str(e.value)
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1())
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
# wait until the beginning of an even minute
wait_until_begin_of_an_even_minute()
# wait until the 10th second of an even minute
# to avoid writing data at the same time backup is taking
time.sleep(10)
write_volume_random_data(volume)
time.sleep(150) # 2.5 minutes
write_volume_random_data(volume)
time.sleep(150) # 2.5 minutes
check_jobs1_result(volume)
volume = volume.detach(hostId="")
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_recurring_job(clients, volume_name): # NOQA
for host_id, client in clients.iteritems(): # NOQA
break
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2)
volume = common.wait_for_volume_detached(client, volume_name)
jobs = create_jobs1()
volume.recurringUpdate(jobs=jobs)
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
job_backup2 = {
"name": "backup2",
"cron": "* * * * *",
"task": "backup",
"retain": 2
}
volume.recurringUpdate(jobs=[jobs[0], job_backup2])
# 5 minutes
time.sleep(300)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot["removed"] is False:
count += 1
# 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head
assert count == 6
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def backup_labels_test(clients,
random_labels,
volume_name,
size=SIZE,
base_image=""): # NOQA
for _, client in clients.iteritems():
break
host_id = get_self_host_id()
volume = create_and_check_volume(client, volume_name, 2, size, base_image)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
bv, b, _, _ = create_backup(client, volume_name, labels=random_labels)
# If we're running the test with a BaseImage, check that this Label is
# set properly.
backup = bv.backupGet(name=b["name"])
if base_image:
assert backup["labels"].get(common.BASE_IMAGE_LABEL) == base_image
# One extra Label from the BaseImage being set.
assert len(backup["labels"]) == len(random_labels) + 1
else:
assert len(backup["labels"]) == len(random_labels)
cleanup_volume(client, volume)
def test_tag_scheduling(client, node_default_tags): # NOQA
"""
Test that scheduling succeeds if there are available Nodes/Disks with the
requested Tags.
"""
host_id = get_self_host_id()
tag_specs = [
# Select all Nodes.
{
"disk": [],
"expected": 3,
"node": []
},
# Selector works with AND on Disk Tags.
{
"disk": ["ssd", "nvme"],
"expected": 2,
"node": []
},
# Selector works with AND on Node Tags.
{
"disk": [],
"expected": 2,
"node": ["main", "storage"]
},
# Selector works based on combined Disk AND Node selector.
{
"disk": ["ssd", "nvme"],
"expected": 1,
"node": ["storage", "main"]
}
]
for specs in tag_specs:
volume_name = generate_volume_name() # NOQA
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3,
diskSelector=specs["disk"],
nodeSelector=specs["node"])
volume = wait_for_volume_detached(client, volume_name)
assert volume["diskSelector"] == specs["disk"]
assert volume["nodeSelector"] == specs["node"]
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
check_volume_replicas(volume, specs, node_default_tags)
cleanup_volume(client, volume)
def test_volume_scheduling_failure(clients, volume_name): # NOQA
'''
Test fail to schedule by disable scheduling for all the nodes
Also test cannot attach a scheduling failed volume
'''
client = get_random_client(clients)
nodes = client.list_node()
assert len(nodes) > 0
for node in nodes:
node = client.update(node, allowScheduling=False)
node = common.wait_for_node_update(client, node["id"],
"allowScheduling", False)
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=3)
volume = common.wait_for_volume_condition_scheduled(client, volume_name,
"status",
CONDITION_STATUS_FALSE)
volume = common.wait_for_volume_detached(client, volume_name)
self_node = get_self_host_id()
with pytest.raises(Exception) as e:
volume.attach(hostId=self_node)
assert "not scheduled" in str(e.value)
for node in nodes:
node = client.update(node, allowScheduling=True)
node = common.wait_for_node_update(client, node["id"],
"allowScheduling", True)
volume = common.wait_for_volume_condition_scheduled(client, volume_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, volume_name)
volume = volume.attach(hostId=self_node)
volume = common.wait_for_volume_healthy(client, volume_name)
endpoint = get_volume_endpoint(volume)
assert endpoint != ""
volume_rw_test(endpoint)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
def test_tag_scheduling_on_update(client, node_default_tags, volume_name): # NOQA
"""
Test that Replicas get scheduled if a Node/Disk disks updated with the
proper Tags.
"""
tag_spec = {
"disk": ["ssd", "m2"],
"expected": 1,
"node": ["main", "fallback"]
}
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3,
diskSelector=tag_spec["disk"],
nodeSelector=tag_spec["node"])
volume = wait_for_volume_detached(client, volume_name)
assert volume["diskSelector"] == tag_spec["disk"]
assert volume["nodeSelector"] == tag_spec["node"]
wait_scheduling_failure(client, volume_name)
host_id = get_self_host_id()
node = client.by_id_node(host_id)
update_disks = get_update_disks(node["disks"])
update_disks[0]["tags"] = tag_spec["disk"]
node = node.diskUpdate(disks=update_disks)
set_node_tags(client, node, tag_spec["node"])
scheduled = False
for i in range(RETRY_COUNTS):
v = client.by_id_volume(volume_name)
if v["conditions"]["scheduled"]["status"] == "True":
scheduled = True
if scheduled:
break
sleep(RETRY_INTERVAL)
assert scheduled
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
nodes = client.list_node()
node_mapping = {node["id"]: {
"disk": get_update_disks(node["disks"])[0]["tags"],
"node": node["tags"]
} for node in nodes}
assert len(volume["replicas"]) == 3
check_volume_replicas(volume, tag_spec, node_mapping)
cleanup_volume(client, volume)
def test_deleting_backup_volume(clients): # NOQA
for host_id, client in clients.iteritems():
break
lht_hostId = get_self_host_id()
volName = generate_volume_name()
volume = create_and_check_volume(client, volName)
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, volName)
bv, _, snap1, _ = create_backup(client, volName)
_, _, snap2, _ = create_backup(client, volName)
bv = client.by_id_backupVolume(volName)
client.delete(bv)
common.wait_for_backup_volume_delete(client, volName)
cleanup_volume(client, volume)
def ha_rebuild_replica_test(client, volname): # NOQA
volume = client.by_id_volume(volname)
assert get_volume_endpoint(volume) == DEV_PATH + volname
assert len(volume["replicas"]) == 2
replica0 = volume["replicas"][0]
assert replica0["name"] != ""
replica1 = volume["replicas"][1]
assert replica1["name"] != ""
data = write_volume_random_data(volume)
volume = volume.replicaRemove(name=replica0["name"])
# wait until we saw a replica starts rebuilding
new_replica_found = False
for i in range(RETRY_COUNTS):
v = client.by_id_volume(volname)
for r in v["replicas"]:
if r["name"] != replica0["name"] and \
r["name"] != replica1["name"]:
new_replica_found = True
break
if new_replica_found:
break
time.sleep(RETRY_INTERVAL)
assert new_replica_found
volume = common.wait_for_volume_healthy(client, volname)
volume = client.by_id_volume(volname)
assert volume["state"] == common.VOLUME_STATE_ATTACHED
assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY
assert len(volume["replicas"]) >= 2
found = False
for replica in volume["replicas"]:
if replica["name"] == replica1["name"]:
found = True
break
assert found
check_volume_data(volume, data)
def ha_rebuild_replica_test(client, volname): # NOQA
volume = client.by_id_volume(volname)
assert get_volume_endpoint(volume) == DEV_PATH + volname
assert len(volume["replicas"]) == 2
replica0 = volume["replicas"][0]
assert replica0["name"] != ""
replica1 = volume["replicas"][1]
assert replica1["name"] != ""
data = write_volume_random_data(volume)
volume = volume.replicaRemove(name=replica0["name"])
# wait until we saw a replica starts rebuilding
new_replica_found = False
for i in range(RETRY_COUNTS):
v = client.by_id_volume(volname)
for r in v["replicas"]:
if r["name"] != replica0["name"] and \
r["name"] != replica1["name"]:
new_replica_found = True
break
if new_replica_found:
break
time.sleep(RETRY_INTERVAL)
assert new_replica_found
volume = common.wait_for_volume_healthy(client, volname)
volume = client.by_id_volume(volname)
assert volume["state"] == common.VOLUME_STATE_ATTACHED
assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY
assert len(volume["replicas"]) >= 2
found = False
for replica in volume["replicas"]:
if replica["name"] == replica1["name"]:
found = True
break
assert found
check_volume_data(volume, data)
def test_recurring_job(clients, volume_name): # NOQA
for host_id, client in clients.iteritems(): # NOQA
break
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2)
volume = common.wait_for_volume_detached(client, volume_name)
jobs = create_jobs1()
volume.recurringUpdate(jobs=jobs)
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
job_backup2 = {"name": "backup2", "cron": "* * * * *",
"task": "backup", "retain": 2}
volume.recurringUpdate(jobs=[jobs[0], job_backup2])
# 5 minutes
time.sleep(300)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot["removed"] is False:
count += 1
# 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head
assert count == 6
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_xfs_pv_existing_volume(client, core_api, pod_manifest): # NOQA
"""
Test create PV with existing XFS filesystem
1. Create a volume
2. Create PV/PVC for the existing volume, specify `xfs` as filesystem
3. Attach the volume to the current node.
4. Format it to `xfs`
5. Create a POD using the volume
FIXME: We should write data in step 4 and validate the data in step 5, make
sure the disk won't be reformatted
"""
volume_name = generate_volume_name()
volume = create_and_check_volume(client, volume_name)
create_pv_for_volume(client, core_api, volume, volume_name, "xfs")
create_pvc_for_volume(client, core_api, volume, volume_name)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
cmd = ['mkfs.xfs', get_volume_endpoint(volume)]
subprocess.check_call(cmd)
volume = volume.detach()
volume = wait_for_volume_detached(client, volume_name)
pod_manifest['spec']['volumes'] = [{
"name": "pod-data",
"persistentVolumeClaim": {
"claimName": volume_name
}
}]
create_and_wait_pod(core_api, pod_manifest)
def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA
"""
Test create volume with recurring jobs
1. Create volume with recurring jobs though Longhorn API
2. Verify the recurring jobs run correctly
"""
for host_id, client in iter(clients.items()): # NOQA
break
set_random_backupstore(client)
# error when creating volume with duplicate jobs
with pytest.raises(Exception) as e:
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1() + create_jobs1())
assert "duplicate job" in str(e.value)
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1())
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def setup_migration_test(clients, volume_name, backing_image=""): # NOQA
"""
Creates a new migratable volume then attaches it to the
current node to write some test data on it.
"""
client = get_random_client(clients)
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=REPLICA_COUNT,
backingImage=backing_image,
accessMode="rwx", migratable=True)
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=common.get_self_host_id())
volume = common.wait_for_volume_healthy(client, volume_name)
# write test data
data = common.write_volume_random_data(volume)
common.check_volume_data(volume, data)
volume.detach(hostId="")
volume = common.wait_for_volume_detached(client, volume_name)
return client, volume, data
def test_attach_without_frontend(clients, volume_name): # NOQA
for host_id, client in clients.iteritems():
break
volume = create_and_check_volume(client, volume_name)
lht_hostId = get_self_host_id()
volume.attach(hostId=lht_hostId, disableFrontend=False)
common.wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume["disableFrontend"] is False
assert volume["frontend"] == "blockdev"
snap1_data = write_volume_random_data(volume)
snap1 = volume.snapshotCreate()
write_volume_random_data(volume)
volume.snapshotCreate()
volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=lht_hostId, disableFrontend=True)
common.wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
engine = get_volume_engine(volume)
assert volume["disableFrontend"] is True
assert volume["frontend"] == "blockdev"
assert engine["endpoint"] == ""
volume.snapshotRevert(name=snap1["name"])
volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=lht_hostId, disableFrontend=False)
common.wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume["disableFrontend"] is False
assert volume["frontend"] == "blockdev"
check_volume_data(volume, snap1_data)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
def test_hard_anti_affinity_scheduling(client, volume_name): # NOQA
"""
Test that volumes with Hard Anti-Affinity work as expected.
With Hard Anti-Affinity, scheduling on nodes with existing replicas should
be forbidden, resulting in "Degraded" state.
"""
volume = create_and_check_volume(client, volume_name)
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
data = write_volume_random_data(volume)
setting = client.by_id_setting(SETTING_REPLICA_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
node = client.by_id_node(host_id)
client.update(node, allowScheduling=False)
host_replica = get_host_replica(volume, host_id)
volume.replicaRemove(name=host_replica["name"])
# Instead of waiting for timeout and lengthening the tests a significant
# amount we can make sure the scheduling isn't working by making sure the
# volume becomes Degraded and reports a scheduling error.
volume = wait_for_volume_degraded(client, volume_name)
wait_scheduling_failure(client, volume_name)
# While there are three replicas that should exist to meet the Volume's
# request, only two of those volumes should actually be Healthy.
assert sum([
1 for replica in volume["replicas"]
if replica["running"] and replica["mode"] == "RW"
]) == 2
# Confirm that the final volume is an unscheduled volume.
assert sum([1 for replica in volume["replicas"]
if not replica["hostId"]]) == 1
# Three replicas in total should still exist.
assert len(volume["replicas"]) == 3
check_volume_data(volume, data)
cleanup_volume(client, volume)
def backupstore_test(client, host_id, volname, size):
bv, b, snap2, data = create_backup(client, volname)
# test restore
restoreName = generate_volume_name()
volume = client.create_volume(name=restoreName,
size=size,
numberOfReplicas=2,
fromBackup=b["url"])
volume = common.wait_for_volume_restoration_completed(client, restoreName)
volume = common.wait_for_volume_detached(client, restoreName)
assert volume["name"] == restoreName
assert volume["size"] == size
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["initialRestorationRequired"] is False
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, restoreName)
check_volume_data(volume, data)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, restoreName)
bv.backupDelete(name=b["name"])
backups = bv.backupList()
found = False
for b in backups:
if b["snapshotName"] == snap2["name"]:
found = True
break
assert not found
volume = wait_for_volume_status(client, volume["name"], "lastBackup", "")
assert volume["lastBackupAt"] == ""
client.delete(volume)
volume = wait_for_volume_delete(client, restoreName)
def migration_rollback_test(clients, volume_name, backing_image=""): # NOQA
client, volume, data = setup_migration_test(clients, volume_name,
backing_image)
host1, host2 = get_hosts_for_migration_test(clients)
volume.attach(hostId=host1)
volume = common.wait_for_volume_healthy(client, volume_name)
volume.attach(hostId=host2)
volume = common.wait_for_volume_migration_ready(client, volume_name)
volume.detach(hostId=host2)
volume = common.wait_for_volume_migration_node(client, volume_name, host1)
volume.detach(hostId="")
volume = common.wait_for_volume_detached(client, volume_name)
# verify test data
check_volume_data(client, volume_name, data)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
def create_volume(client, vol_name, size, node_id, r_num): # NOQA
volume = client.create_volume(name=vol_name, size=size,
numberOfReplicas=r_num)
assert volume["numberOfReplicas"] == r_num
assert volume["frontend"] == "blockdev"
volume = common.wait_for_volume_detached(client, vol_name)
assert len(volume["replicas"]) == r_num
assert volume["state"] == "detached"
assert volume["created"] != ""
volumeByName = client.by_id_volume(vol_name)
assert volumeByName["name"] == volume["name"]
assert volumeByName["size"] == volume["size"]
assert volumeByName["numberOfReplicas"] == volume["numberOfReplicas"]
assert volumeByName["state"] == volume["state"]
assert volumeByName["created"] == volume["created"]
volume.attach(hostId=node_id)
volume = common.wait_for_volume_healthy(client, vol_name)
return volume
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA
volume = client.create_volume(name=volume_name, size=size,
numberOfReplicas=2, baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
data = write_volume_random_data(volume)
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
_, b = common.find_backup(client, volume_name, snap2["name"])
res_name = common.generate_volume_name()
res_volume = client.create_volume(name=res_name, size=size,
numberOfReplicas=2,
fromBackup=b["url"])
res_volume = common.wait_for_volume_detached(client, res_name)
res_volume = res_volume.attach(hostId=host_id)
res_volume = common.wait_for_volume_healthy(client, res_name)
check_volume_data(res_volume, data)
snapshots = res_volume.snapshotList()
# only the backup snapshot + volume-head
assert len(snapshots) == 2
backup_snapshot = ""
for snap in snapshots:
if snap["name"] != "volume-head":
backup_snapshot = snap["name"]
assert backup_snapshot != ""
res_volume.snapshotCreate()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 3
res_volume.snapshotDelete(name=backup_snapshot)
res_volume.snapshotPurge()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 2
ha_rebuild_replica_test(client, res_name)
res_volume = res_volume.detach()
res_volume = common.wait_for_volume_detached(client, res_name)
client.delete(res_volume)
common.wait_for_volume_delete(client, res_name)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_replica_scheduler_update_minimal_available(client): # NOQA
minimal_available_setting = client.by_id_setting(
SETTING_STORAGE_MINIMAL_AVAILABLE_PERCENTAGE)
old_minimal_setting = minimal_available_setting["value"]
nodes = client.list_node()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
# set storage minimal available percentage to 100
# to test all replica couldn't be scheduled
minimal_available_setting = client.update(minimal_available_setting,
value="100")
# wait for disks state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"],
fsid, DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_FALSE)
lht_hostId = get_self_host_id()
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE, numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
# set storage minimal available percentage to default value(10)
minimal_available_setting = client.update(minimal_available_setting,
value=old_minimal_setting)
# wait for disks state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"],
fsid, DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_TRUE)
# check volume status
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
def test_replica_scheduler_just_under_over_provisioning(client): # NOQA
over_provisioning_setting = client.by_id_setting(
SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE)
old_provisioning_setting = over_provisioning_setting["value"]
# set storage over provisioning percentage to 100
over_provisioning_setting = client.update(over_provisioning_setting,
value="100")
lht_hostId = get_self_host_id()
nodes = client.list_node()
expect_node_disk = {}
max_size_array = []
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
max_size_array.append(disk["storageMaximum"])
disk["storageReserved"] = 0
update_disks = get_update_disks(disks)
node = node.diskUpdate(disks=update_disks)
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_status(client, node["name"],
fsid, "storageReserved", 0)
max_size = min(max_size_array)
# test just under over provisioning limit could be scheduled
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=str(max_size),
numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
client.update(over_provisioning_setting, value=old_provisioning_setting)
def test_replica_scheduler_too_large_volume_fit_any_disks(client): # NOQA
nodes = client.list_node()
lht_hostId = get_self_host_id()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
disk["storageReserved"] = disk["storageMaximum"]
update_disks = get_update_disks(disks)
node.diskUpdate(disks=update_disks)
# volume is too large to fill into any disks
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=str(4*Gi), numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
# reduce StorageReserved of each default disk
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
update_disks = get_update_disks(disks)
for disk in update_disks:
disk["storageReserved"] = 0
node = node.diskUpdate(disks=update_disks)
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_status(client, node["name"],
fsid, "storageReserved", 0)
# check volume status
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
