def test_node_controller_sync_disk_state(client): # NOQA
# update StorageMinimalAvailablePercentage to test Disk State
setting = client.by_id_setting(
SETTING_STORAGE_MINIMAL_AVAILABLE_PERCENTAGE)
old_minimal_available_percentage = setting["value"]
setting = client.update(setting, value="100")
assert setting["value"] == "100"
nodes = client.list_node()
# wait for node controller to update disk state
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"], fsid,
DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_FALSE)
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_FALSE
setting = client.update(setting, value=old_minimal_available_percentage)
assert setting["value"] == old_minimal_available_percentage
# wait for node controller to update disk state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"], fsid,
DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_TRUE)
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_TRUE
def test_node_controller_sync_disk_state(client): # NOQA
# update StorageMinimalAvailablePercentage to test Disk State
setting = client.by_id_setting(
SETTING_STORAGE_MINIMAL_AVAILABLE_PERCENTAGE)
old_minimal_available_percentage = setting["value"]
setting = client.update(setting, value="100")
assert setting["value"] == "100"
nodes = client.list_node()
# wait for node controller to update disk state
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"],
fsid, DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_FALSE)
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_FALSE
setting = client.update(setting, value=old_minimal_available_percentage)
assert setting["value"] == old_minimal_available_percentage
# wait for node controller to update disk state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"],
fsid, DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_TRUE)
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_TRUE
def test_node_delete_umount_disks(client): # NOQA
# create test disks for node
disk_volume_name = 'vol-disk-1'
lht_hostId = get_self_host_id()
node = client.by_id_node(lht_hostId)
disks = node["disks"]
disk_path1 = create_host_disk(client, disk_volume_name, str(Gi),
lht_hostId)
disk1 = {
"path": disk_path1,
"allowScheduling": True,
"storageReserved": SMALL_DISK_SIZE
}
update_disk = get_update_disks(disks)
for disk in update_disk:
disk["allowScheduling"] = False
# add new disk for node
update_disk.append(disk1)
# save disks to node
node = node.diskUpdate(disks=update_disk)
node = common.wait_for_disk_update(client, lht_hostId, len(update_disk))
assert len(node["disks"]) == len(update_disk)
node = client.by_id_node(lht_hostId)
assert len(node["disks"]) == len(update_disk)
disks = node["disks"]
# wait for node controller to update disk status
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling",
True)
wait_for_disk_status(client, lht_hostId, fsid, "storageReserved",
SMALL_DISK_SIZE)
free, total = common.get_host_disk_size(disk_path1)
wait_for_disk_status(client, lht_hostId, fsid, "storageAvailable",
free)
wait_for_disk_status(client, lht_hostId, fsid, "storageMaximum",
total)
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for key, disk in disks.iteritems():
if disk["path"] == disk_path1:
assert disk["allowScheduling"]
assert disk["storageReserved"] == SMALL_DISK_SIZE
assert disk["storageScheduled"] == 0
free, total = common.get_host_disk_size(disk_path1)
assert disk["storageMaximum"] == total
assert disk["storageAvailable"] == free
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_READY]["status"] == \
CONDITION_STATUS_TRUE
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_TRUE
else:
assert not disk["allowScheduling"]
# create a volume
nodes = client.list_node()
vol_name = common.generate_volume_name()
volume = create_volume(client, vol_name, str(SMALL_DISK_SIZE), lht_hostId,
len(nodes))
replicas = volume["replicas"]
for replica in replicas:
id = replica["hostId"]
assert id != ""
assert replica["running"]
if id == lht_hostId:
assert replica["dataPath"].startswith(disk_path1)
# umount the disk
mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name)
common.umount_disk(mount_path)
# wait for update node status
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling",
False)
wait_for_disk_status(client, lht_hostId, fsid, "storageMaximum", 0)
wait_for_disk_conditions(client, lht_hostId, fsid,
DISK_CONDITION_READY,
CONDITION_STATUS_FALSE)
# check result
node = client.by_id_node(lht_hostId)
disks = node["disks"]
update_disks = []
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
assert not disk["allowScheduling"]
assert disk["storageMaximum"] == 0
assert disk["storageAvailable"] == 0
assert disk["storageReserved"] == SMALL_DISK_SIZE
assert disk["storageScheduled"] == SMALL_DISK_SIZE
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_READY]["status"] == \
CONDITION_STATUS_FALSE
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_FALSE
else:
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_READY]["status"] == \
CONDITION_STATUS_TRUE
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_TRUE
update_disks.append(disk)
# delete umount disk exception
with pytest.raises(Exception) as e:
node.diskUpdate(disks=update_disks)
assert "disable the disk" in str(e.value)
# update other disks
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] != disk_path1:
disk["allowScheduling"] = True
test_update = get_update_disks(disks)
node = node.diskUpdate(disks=test_update)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] != disk_path1:
wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling",
True)
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] != disk_path1:
assert disk["allowScheduling"]
# mount the disk back
mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name)
disk_volume = client.by_id_volume(disk_volume_name)
dev = get_volume_endpoint(disk_volume)
common.mount_disk(dev, mount_path)
# wait for update node status
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling",
False)
wait_for_disk_conditions(client, lht_hostId, fsid,
DISK_CONDITION_READY,
CONDITION_STATUS_TRUE)
# check result
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
free, total = common.get_host_disk_size(disk_path1)
assert not disk["allowScheduling"]
assert disk["storageMaximum"] == total
assert disk["storageAvailable"] == free
assert disk["storageReserved"] == SMALL_DISK_SIZE
assert disk["storageScheduled"] == SMALL_DISK_SIZE
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_READY]["status"] == \
CONDITION_STATUS_TRUE
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_TRUE
else:
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_READY]["status"] == \
CONDITION_STATUS_TRUE
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_TRUE
# delete volume and umount disk
cleanup_volume(client, vol_name)
mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name)
common.umount_disk(mount_path)
# wait for update node status
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
wait_for_disk_status(client, lht_hostId, fsid, "allowScheduling",
False)
wait_for_disk_status(client, lht_hostId, fsid, "storageScheduled",
0)
wait_for_disk_status(client, lht_hostId, fsid, "storageMaximum", 0)
# test delete the umount disk
node = client.by_id_node(lht_hostId)
node.diskUpdate(disks=update_disks)
node = common.wait_for_disk_update(client, lht_hostId, len(update_disks))
assert len(node["disks"]) == len(update_disks)
cmd = ['rm', '-r', mount_path]
subprocess.check_call(cmd)
def test_replica_scheduler_update_minimal_available(client): # NOQA
minimal_available_setting = client.by_id_setting(
SETTING_STORAGE_MINIMAL_AVAILABLE_PERCENTAGE)
old_minimal_setting = minimal_available_setting["value"]
nodes = client.list_node()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
# set storage minimal available percentage to 100
# to test all replica couldn't be scheduled
minimal_available_setting = client.update(minimal_available_setting,
value="100")
# wait for disks state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"], fsid,
DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_FALSE)
lht_hostId = get_self_host_id()
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE,
numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(
client, vol_name, "status", CONDITION_STATUS_FALSE)
# set storage minimal available percentage to default value(10)
minimal_available_setting = client.update(minimal_available_setting,
value=old_minimal_setting)
# wait for disks state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"], fsid,
DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_TRUE)
# check volume status
volume = common.wait_for_volume_condition_scheduled(
client, vol_name, "status", CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
def test_node_delete_umount_disks(client): # NOQA
# create test disks for node
disk_volume_name = 'vol-disk-1'
lht_hostId = get_self_host_id()
node = client.by_id_node(lht_hostId)
disks = node["disks"]
disk_path1 = create_host_disk(client, disk_volume_name,
str(Gi), lht_hostId)
disk1 = {"path": disk_path1, "allowScheduling": True,
"storageReserved": SMALL_DISK_SIZE}
update_disk = get_update_disks(disks)
for disk in update_disk:
disk["allowScheduling"] = False
# add new disk for node
update_disk.append(disk1)
# save disks to node
node = node.diskUpdate(disks=update_disk)
node = common.wait_for_disk_update(client, lht_hostId,
len(update_disk))
assert len(node["disks"]) == len(update_disk)
node = client.by_id_node(lht_hostId)
assert len(node["disks"]) == len(update_disk)
disks = node["disks"]
# wait for node controller to update disk status
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
wait_for_disk_status(client, lht_hostId, fsid,
"allowScheduling", True)
wait_for_disk_status(client, lht_hostId, fsid,
"storageReserved", SMALL_DISK_SIZE)
free, total = common.get_host_disk_size(disk_path1)
wait_for_disk_status(client, lht_hostId, fsid,
"storageAvailable", free)
wait_for_disk_status(client, lht_hostId, fsid,
"storageMaximum", total)
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for key, disk in disks.iteritems():
if disk["path"] == disk_path1:
assert disk["allowScheduling"]
assert disk["storageReserved"] == SMALL_DISK_SIZE
assert disk["storageScheduled"] == 0
free, total = common.get_host_disk_size(disk_path1)
assert disk["storageMaximum"] == total
assert disk["storageAvailable"] == free
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_READY]["status"] == \
CONDITION_STATUS_TRUE
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_TRUE
else:
assert not disk["allowScheduling"]
# create a volume
nodes = client.list_node()
vol_name = common.generate_volume_name()
volume = create_volume(client, vol_name, str(SMALL_DISK_SIZE),
lht_hostId, len(nodes))
replicas = volume["replicas"]
for replica in replicas:
id = replica["hostId"]
assert id != ""
assert replica["running"]
if id == lht_hostId:
assert replica["dataPath"].startswith(disk_path1)
# umount the disk
mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name)
common.umount_disk(mount_path)
# wait for update node status
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
wait_for_disk_status(client, lht_hostId,
fsid, "allowScheduling", False)
wait_for_disk_status(client, lht_hostId,
fsid, "storageMaximum", 0)
wait_for_disk_conditions(client, lht_hostId, fsid,
DISK_CONDITION_READY,
CONDITION_STATUS_FALSE)
# check result
node = client.by_id_node(lht_hostId)
disks = node["disks"]
update_disks = []
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
assert not disk["allowScheduling"]
assert disk["storageMaximum"] == 0
assert disk["storageAvailable"] == 0
assert disk["storageReserved"] == SMALL_DISK_SIZE
assert disk["storageScheduled"] == SMALL_DISK_SIZE
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_READY]["status"] == \
CONDITION_STATUS_FALSE
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_FALSE
else:
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_READY]["status"] == \
CONDITION_STATUS_TRUE
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_TRUE
update_disks.append(disk)
# delete umount disk exception
with pytest.raises(Exception) as e:
node.diskUpdate(disks=update_disks)
assert "disable the disk" in str(e.value)
# update other disks
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] != disk_path1:
disk["allowScheduling"] = True
test_update = get_update_disks(disks)
node = node.diskUpdate(disks=test_update)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] != disk_path1:
wait_for_disk_status(client, lht_hostId,
fsid, "allowScheduling", True)
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] != disk_path1:
assert disk["allowScheduling"]
# mount the disk back
mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name)
disk_volume = client.by_id_volume(disk_volume_name)
dev = get_volume_endpoint(disk_volume)
common.mount_disk(dev, mount_path)
# wait for update node status
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
wait_for_disk_status(client, lht_hostId,
fsid, "allowScheduling", False)
wait_for_disk_conditions(client, lht_hostId, fsid,
DISK_CONDITION_READY,
CONDITION_STATUS_TRUE)
# check result
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
free, total = common.get_host_disk_size(disk_path1)
assert not disk["allowScheduling"]
assert disk["storageMaximum"] == total
assert disk["storageAvailable"] == free
assert disk["storageReserved"] == SMALL_DISK_SIZE
assert disk["storageScheduled"] == SMALL_DISK_SIZE
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_READY]["status"] == \
CONDITION_STATUS_TRUE
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_TRUE
else:
conditions = disk["conditions"]
assert conditions[DISK_CONDITION_READY]["status"] == \
CONDITION_STATUS_TRUE
assert conditions[DISK_CONDITION_SCHEDULABLE]["status"] == \
CONDITION_STATUS_TRUE
# delete volume and umount disk
cleanup_volume(client, vol_name)
mount_path = os.path.join(DIRECTORY_PATH, disk_volume_name)
common.umount_disk(mount_path)
# wait for update node status
node = client.by_id_node(lht_hostId)
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == disk_path1:
wait_for_disk_status(client, lht_hostId,
fsid, "allowScheduling", False)
wait_for_disk_status(client, lht_hostId,
fsid, "storageScheduled", 0)
wait_for_disk_status(client, lht_hostId,
fsid, "storageMaximum", 0)
# test delete the umount disk
node = client.by_id_node(lht_hostId)
node.diskUpdate(disks=update_disks)
node = common.wait_for_disk_update(client, lht_hostId,
len(update_disks))
assert len(node["disks"]) == len(update_disks)
cmd = ['rm', '-r', mount_path]
subprocess.check_call(cmd)
def test_replica_scheduler_update_minimal_available(client): # NOQA
minimal_available_setting = client.by_id_setting(
SETTING_STORAGE_MINIMAL_AVAILABLE_PERCENTAGE)
old_minimal_setting = minimal_available_setting["value"]
nodes = client.list_node()
expect_node_disk = {}
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
if disk["path"] == DEFAULT_DISK_PATH:
expect_disk = disk
expect_disk["fsid"] = fsid
expect_node_disk[node["name"]] = expect_disk
# set storage minimal available percentage to 100
# to test all replica couldn't be scheduled
minimal_available_setting = client.update(minimal_available_setting,
value="100")
# wait for disks state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"],
fsid, DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_FALSE)
lht_hostId = get_self_host_id()
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE, numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
# set storage minimal available percentage to default value(10)
minimal_available_setting = client.update(minimal_available_setting,
value=old_minimal_setting)
# wait for disks state
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_conditions(client, node["name"],
fsid, DISK_CONDITION_SCHEDULABLE,
CONDITION_STATUS_TRUE)
# check volume status
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_TRUE)
volume = common.wait_for_volume_detached(client, vol_name)
assert volume["state"] == "detached"
assert volume["created"] != ""
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, vol_name)
nodes = client.list_node()
node_hosts = []
for node in nodes:
node_hosts.append(node["name"])
# check all replica should be scheduled to default disk
for replica in volume["replicas"]:
id = replica["hostId"]
assert id != ""
assert replica["running"]
expect_disk = expect_node_disk[id]
assert replica["diskID"] == expect_disk["fsid"]
assert expect_disk["path"] in replica["dataPath"]
node_hosts = filter(lambda x: x != id, node_hosts)
assert len(node_hosts) == 0
# clean volume and disk
cleanup_volume(client, vol_name)
