def test_replica_zone_anti_affinity(client, core_api, volume_name,
k8s_node_zone_tags): # NOQA
"""
Test replica scheduler with zone anti-affinity
1. Set zone anti-affinity to hard.
2. Label nodes 1 & 2 with same zone label "zone1".
Label node 3 with zone label "zone2".
3. Create a volume with 3 replicas.
4. Wait for volume condition `scheduled` to be false.
5. Label node 2 with zone label "zone3".
6. Wait for volume condition `scheduled` to be success.
7. Clear the volume.
8. Set zone anti-affinity to soft.
9. Change the zone labels on node 1 & 2 & 3 to "zone1".
10. Create a volume.
11. Wait for volume condition `scheduled` to be success.
12. Clean up the replica count, the zone labels and the volume.
"""
wait_longhorn_node_zone_updated(client)
replica_node_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(replica_node_soft_anti_affinity_setting, value="false")
replica_zone_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_ZONE_SOFT_ANTI_AFFINITY)
client.update(replica_zone_soft_anti_affinity_setting, value="false")
volume = create_and_check_volume(client, volume_name)
lh_nodes = client.list_node()
count = 0
for node in lh_nodes:
count += 1
set_k8s_node_zone_label(core_api, node.name, "lh-zone" + str(count))
wait_longhorn_node_zone_updated(client)
wait_for_volume_condition_scheduled(client, volume_name, "status",
CONDITION_STATUS_TRUE)
replica_zone_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_ZONE_SOFT_ANTI_AFFINITY)
client.update(replica_zone_soft_anti_affinity_setting, value="true")
volume = client.by_id_volume(volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
for node in lh_nodes:
set_k8s_node_zone_label(core_api, node.name, "lh-zone1")
wait_longhorn_node_zone_updated(client)
volume = create_and_check_volume(client, volume_name)
wait_for_volume_condition_scheduled(client, volume_name, "status",
CONDITION_STATUS_TRUE)
def test_replica_scheduler_update_over_provisioning(client): # NOQA
nodes = client.list_node()
lht_hostId = get_self_host_id()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
over_provisioning_setting = client.by_id_setting(
SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE)
old_provisioning_setting = over_provisioning_setting["value"]
# set storage over provisioning percentage to 0
# to test all replica couldn't be scheduled
over_provisioning_setting = client.update(over_provisioning_setting,
value="0")
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE, numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
# set storage over provisioning percentage to 100
over_provisioning_setting = client.update(over_provisioning_setting,
value="100")
# check volume status
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
client.update(over_provisioning_setting,
value=old_provisioning_setting)
def test_replica_scheduler_update_over_provisioning(client): # NOQA
nodes = client.list_node()
lht_hostId = get_self_host_id()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
over_provisioning_setting = client.by_id_setting(
SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE)
old_provisioning_setting = over_provisioning_setting["value"]
# set storage over provisioning percentage to 0
# to test all replica couldn't be scheduled
over_provisioning_setting = client.update(over_provisioning_setting,
value="0")
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE, numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
# set storage over provisioning percentage to 100
over_provisioning_setting = client.update(over_provisioning_setting,
value="100")
# check volume status
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
client.update(over_provisioning_setting,
value=old_provisioning_setting)
def test_replica_scheduler_just_under_over_provisioning(client): # NOQA
over_provisioning_setting = client.by_id_setting(
SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE)
old_provisioning_setting = over_provisioning_setting["value"]
# set storage over provisioning percentage to 100
over_provisioning_setting = client.update(over_provisioning_setting,
value="100")
lht_hostId = get_self_host_id()
nodes = client.list_node()
expect_node_disk = {}
max_size_array = []
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
max_size_array.append(disk["storageMaximum"])
disk["storageReserved"] = 0
update_disks = get_update_disks(disks)
node = node.diskUpdate(disks=update_disks)
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_status(client, node["name"], fsid,
"storageReserved", 0)
max_size = min(max_size_array)
# test just under over provisioning limit could be scheduled
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=str(max_size),
numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(
client, vol_name, "status", CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
client.update(over_provisioning_setting, value=old_provisioning_setting)
def test_replica_scheduler_exceed_over_provisioning(client): # NOQA
over_provisioning_setting = client.by_id_setting(
SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE)
old_provisioning_setting = over_provisioning_setting["value"]
# set storage over provisioning percentage to 100
over_provisioning_setting = client.update(over_provisioning_setting,
value="100")
# test exceed over provisioning limit couldn't be scheduled
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
disk["storageReserved"] = \
disk["storageMaximum"] - 1*Gi
update_disks = get_update_disks(disks)
node = node.diskUpdate(disks=update_disks)
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_status(client, node["name"], fsid, "storageReserved",
disk["storageMaximum"] - 1 * Gi)
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=str(2 * Gi),
numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(
client, vol_name, "status", CONDITION_STATUS_FALSE)
client.delete(volume)
common.wait_for_volume_delete(client, vol_name)
client.update(over_provisioning_setting, value=old_provisioning_setting)
def test_replica_scheduler_no_disks(client): # NOQA
nodes = client.list_node()
# delete all disks on each node
for node in nodes:
disks = node["disks"]
name = node["name"]
# set allowScheduling to false
for fsid, disk in disks.iteritems():
disk["allowScheduling"] = False
update_disks = get_update_disks(disks)
node = node.diskUpdate(disks=update_disks)
for fsid, disk in node["disks"].iteritems():
# wait for node controller update disk status
wait_for_disk_status(client, name, fsid, "allowScheduling", False)
wait_for_disk_status(client, name, fsid, "storageScheduled", 0)
node = client.by_id_node(name)
for fsid, disk in node["disks"].iteritems():
assert not disk["allowScheduling"]
node = node.diskUpdate(disks=[])
node = common.wait_for_disk_update(client, name, 0)
assert len(node["disks"]) == 0
# test there's no disk fit for volume
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE,
numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(
client, vol_name, "status", CONDITION_STATUS_FALSE)
client.delete(volume)
common.wait_for_volume_delete(client, vol_name)
def test_replica_scheduler_exceed_over_provisioning(client): # NOQA
over_provisioning_setting = client.by_id_setting(
SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE)
old_provisioning_setting = over_provisioning_setting["value"]
# set storage over provisioning percentage to 100
over_provisioning_setting = client.update(over_provisioning_setting,
value="100")
# test exceed over provisioning limit couldn't be scheduled
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
disk["storageReserved"] = \
disk["storageMaximum"] - 1*Gi
update_disks = get_update_disks(disks)
node = node.diskUpdate(disks=update_disks)
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_status(client, node["name"],
fsid, "storageReserved",
disk["storageMaximum"] - 1*Gi)
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=str(2*Gi), numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
client.delete(volume)
common.wait_for_volume_delete(client, vol_name)
client.update(over_provisioning_setting, value=old_provisioning_setting)
def test_replica_scheduler_no_disks(client): # NOQA
nodes = client.list_node()
# delete all disks on each node
for node in nodes:
disks = node["disks"]
name = node["name"]
# set allowScheduling to false
for fsid, disk in disks.iteritems():
disk["allowScheduling"] = False
update_disks = get_update_disks(disks)
node = node.diskUpdate(disks=update_disks)
for fsid, disk in node["disks"].iteritems():
# wait for node controller update disk status
wait_for_disk_status(client, name, fsid,
"allowScheduling", False)
wait_for_disk_status(client, name, fsid,
"storageScheduled", 0)
node = client.by_id_node(name)
for fsid, disk in node["disks"].iteritems():
assert not disk["allowScheduling"]
node = node.diskUpdate(disks=[])
node = common.wait_for_disk_update(client, name, 0)
assert len(node["disks"]) == 0
# test there's no disk fit for volume
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE, numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
client.delete(volume)
common.wait_for_volume_delete(client, vol_name)
def test_volume_scheduling_failure(clients, volume_name): # NOQA
'''
Test fail to schedule by disable scheduling for all the nodes
Also test cannot attach a scheduling failed volume
'''
client = get_random_client(clients)
nodes = client.list_node()
assert len(nodes) > 0
for node in nodes:
node = client.update(node, allowScheduling=False)
node = common.wait_for_node_update(client, node["id"],
"allowScheduling", False)
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=3)
volume = common.wait_for_volume_condition_scheduled(client, volume_name,
"status",
CONDITION_STATUS_FALSE)
volume = common.wait_for_volume_detached(client, volume_name)
self_node = get_self_host_id()
with pytest.raises(Exception) as e:
volume.attach(hostId=self_node)
assert "not scheduled" in str(e.value)
for node in nodes:
node = client.update(node, allowScheduling=True)
node = common.wait_for_node_update(client, node["id"],
"allowScheduling", True)
volume = common.wait_for_volume_condition_scheduled(client, volume_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, volume_name)
volume = volume.attach(hostId=self_node)
volume = common.wait_for_volume_healthy(client, volume_name)
endpoint = get_volume_endpoint(volume)
assert endpoint != ""
volume_rw_test(endpoint)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
def test_allow_volume_creation_with_degraded_availability_csi(
client, core_api, apps_api, make_deployment_with_pvc): # NOQA
"""
Test Allow Volume Creation with Degraded Availability (CSI)
Requirement:
1. Set `allow-volume-creation-with-degraded-availability` to true.
2. Set `node-level-soft-anti-affinity` to false.
Steps:
1. Disable scheduling for node 3.
2. Create a Deployment Pod with a volume and 3 replicas.
1. After the volume is attached, scheduling error should be seen.
3. Write data to the Pod.
4. Scale down the deployment to 0 to detach the volume.
1. Scheduled condition should become true.
5. Scale up the deployment back to 1 and verify the data.
1. Scheduled condition should become false.
6. Enable the scheduling for node 3.
1. Volume should start rebuilding on the node 3 soon.
2. Once the rebuilding starts, the scheduled condition should become
true.
7. Once rebuild finished, scale down and back the deployment to verify
the data.
"""
setting = client.by_id_setting(common.SETTING_DEGRADED_AVAILABILITY)
client.update(setting, value="true")
setting = client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(setting, value="false")
nodes = client.list_node()
node3 = nodes[2]
client.update(node3, allowScheduling=False)
vol = common.create_and_check_volume(client, generate_volume_name(),
size=str(500 * Mi))
pv_name = vol.name + "-pv"
common.create_pv_for_volume(client, core_api, vol, pv_name)
pvc_name = vol.name + "-pvc"
common.create_pvc_for_volume(client, core_api, vol, pvc_name)
deployment_name = vol.name + "-dep"
deployment = make_deployment_with_pvc(deployment_name, pvc_name)
deployment["spec"]["replicas"] = 3
apps_api.create_namespaced_deployment(body=deployment, namespace='default')
common.wait_for_volume_status(client, vol.name,
common.VOLUME_FIELD_STATE,
common.VOLUME_STATE_ATTACHED)
common.wait_scheduling_failure(client, vol.name)
data_path = "/data/test"
pod = common.wait_and_get_any_deployment_pod(core_api, deployment_name)
common.write_pod_volume_random_data(core_api, pod.metadata.name,
data_path, common.DATA_SIZE_IN_MB_2)
created_md5sum = get_pod_data_md5sum(core_api, pod.metadata.name,
data_path)
deployment['spec']['replicas'] = 0
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment_name)
vol = common.wait_for_volume_detached(client, vol.name)
assert vol.conditions[VOLUME_CONDITION_SCHEDULED]['status'] == "True"
deployment['spec']['replicas'] = 1
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment_name)
common.wait_for_volume_status(client, vol.name,
common.VOLUME_FIELD_STATE,
common.VOLUME_STATE_ATTACHED)
common.wait_for_volume_condition_scheduled(client, vol.name, "status",
common.CONDITION_STATUS_FALSE)
pod = common.wait_and_get_any_deployment_pod(core_api, deployment_name)
assert created_md5sum == get_pod_data_md5sum(core_api,
pod.metadata.name,
data_path)
client.update(node3, allowScheduling=True)
common.wait_for_rebuild_start(client, vol.name)
vol = client.by_id_volume(vol.name)
assert vol.conditions[VOLUME_CONDITION_SCHEDULED]['status'] == "True"
common.wait_for_rebuild_complete(client, vol.name)
deployment['spec']['replicas'] = 0
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment_name)
common.wait_for_volume_detached(client, vol.name)
deployment['spec']['replicas'] = 1
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment_name)
common.wait_for_volume_status(client, vol.name,
common.VOLUME_FIELD_STATE,
common.VOLUME_STATE_ATTACHED)
pod = common.wait_and_get_any_deployment_pod(core_api, deployment_name)
assert created_md5sum == get_pod_data_md5sum(core_api,
pod.metadata.name,
data_path)
def test_replica_scheduler_update_minimal_available(client): # NOQA
minimal_available_setting = client.by_id_setting(
SETTING_STORAGE_MINIMAL_AVAILABLE_PERCENTAGE)
old_minimal_setting = minimal_available_setting["value"]
nodes = client.list_node()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
# set storage minimal available percentage to 100
# to test all replica couldn't be scheduled
minimal_available_setting = client.update(minimal_available_setting,
value="100")
# wait for disks state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"], fsid,
DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_FALSE)
lht_hostId = get_self_host_id()
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE,
numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(
client, vol_name, "status", CONDITION_STATUS_FALSE)
# set storage minimal available percentage to default value(10)
minimal_available_setting = client.update(minimal_available_setting,
value=old_minimal_setting)
# wait for disks state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"], fsid,
DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_TRUE)
# check volume status
volume = common.wait_for_volume_condition_scheduled(
client, vol_name, "status", CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
def test_replica_scheduler_too_large_volume_fit_any_disks(client): # NOQA
nodes = client.list_node()
lht_hostId = get_self_host_id()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
disk["storageReserved"] = disk["storageMaximum"]
update_disks = get_update_disks(disks)
node.diskUpdate(disks=update_disks)
# volume is too large to fill into any disks
volume_size = 4 * Gi
vol_name = common.generate_volume_name()
client.create_volume(name=vol_name,
size=str(volume_size),
numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(
client, vol_name, "status", CONDITION_STATUS_FALSE)
# Reduce StorageReserved of each default disk so that each node can fit
# only one replica.
needed_for_scheduling = int(
volume_size * 1.5 * 100 /
int(DEFAULT_STORAGE_OVER_PROVISIONING_PERCENTAGE))
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
update_disks = get_update_disks(disks)
for disk in update_disks:
disk["storageReserved"] = \
disk["storageMaximum"] - needed_for_scheduling
node = node.diskUpdate(disks=update_disks)
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_status(
client, node["name"], fsid, "storageReserved",
disk["storageMaximum"] - needed_for_scheduling)
# check volume status
volume = common.wait_for_volume_condition_scheduled(
client, vol_name, "status", CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
def test_replica_scheduler_update_minimal_available(client): # NOQA
minimal_available_setting = client.by_id_setting(
SETTING_STORAGE_MINIMAL_AVAILABLE_PERCENTAGE)
old_minimal_setting = minimal_available_setting["value"]
nodes = client.list_node()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
# set storage minimal available percentage to 100
# to test all replica couldn't be scheduled
minimal_available_setting = client.update(minimal_available_setting,
value="100")
# wait for disks state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"],
fsid, DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_FALSE)
lht_hostId = get_self_host_id()
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE, numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
# set storage minimal available percentage to default value(10)
minimal_available_setting = client.update(minimal_available_setting,
value=old_minimal_setting)
# wait for disks state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"],
fsid, DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_TRUE)
# check volume status
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
def test_replica_scheduler_just_under_over_provisioning(client): # NOQA
over_provisioning_setting = client.by_id_setting(
SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE)
old_provisioning_setting = over_provisioning_setting["value"]
# set storage over provisioning percentage to 100
over_provisioning_setting = client.update(over_provisioning_setting,
value="100")
lht_hostId = get_self_host_id()
nodes = client.list_node()
expect_node_disk = {}
max_size_array = []
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
max_size_array.append(disk["storageMaximum"])
disk["storageReserved"] = 0
update_disks = get_update_disks(disks)
node = node.diskUpdate(disks=update_disks)
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_status(client, node["name"],
fsid, "storageReserved", 0)
max_size = min(max_size_array)
# test just under over provisioning limit could be scheduled
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=str(max_size),
numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
client.update(over_provisioning_setting, value=old_provisioning_setting)
def test_replica_scheduler_too_large_volume_fit_any_disks(client): # NOQA
nodes = client.list_node()
lht_hostId = get_self_host_id()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
disk["storageReserved"] = disk["storageMaximum"]
update_disks = get_update_disks(disks)
node.diskUpdate(disks=update_disks)
# volume is too large to fill into any disks
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=str(4*Gi), numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
# reduce StorageReserved of each default disk
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
update_disks = get_update_disks(disks)
for disk in update_disks:
disk["storageReserved"] = 0
node = node.diskUpdate(disks=update_disks)
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_status(client, node["name"],
fsid, "storageReserved", 0)
# check volume status
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
