def test_flexvolume_tags(client, core_api, node_default_tags, flexvolume,
pod): # NOQA
"""
Test that the FlexVolume provisioner is able to handle diskSelectors and
nodeSelectors properly.
"""
pod_name = 'flexvolume-tag-test'
tag_spec = {
"disk": ["ssd", "nvme"],
"expected": 1,
"node": ["storage", "main"]
}
volume_size = DEFAULT_VOLUME_SIZE * Gi
flexvolume["flexvolume"]["options"]["diskSelector"] = "ssd,nvme"
flexvolume["flexvolume"]["options"]["nodeSelector"] = "storage,main"
pod['metadata']['name'] = pod_name
pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \
flexvolume['name']
pod['spec']['volumes'] = [flexvolume]
create_and_wait_pod(core_api, pod)
volumes = client.list_volume()
assert len(volumes) == 1
assert volumes[0]["name"] == flexvolume['name']
assert volumes[0]["size"] == str(volume_size)
assert volumes[0]["numberOfReplicas"] == int(
flexvolume["flexVolume"]["options"]["numberOfReplicas"])
assert volumes[0]["state"] == "attached"
check_volume_replicas(volumes[0], tag_spec, node_default_tags)
def test_provisioner_tags(client, core_api, node_default_tags, storage_class,
pvc, pod): # NOQA
"""
Test that a StorageClass can properly provision a volume with requested
Tags.
Test prerequisite:
- set Replica Node Level Soft Anti-Affinity enabled
1. Use `node_default_tags` to add default tags to nodes.
2. Create a StorageClass with disk and node tag set.
3. Create PVC and Pod.
4. Verify the volume has the correct parameters and tags.
"""
replica_node_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(replica_node_soft_anti_affinity_setting, value="true")
# Prepare pod and volume specs.
pod_name = 'provisioner-tags-test'
tag_spec = {
"disk": ["ssd", "nvme"],
"expected": 1,
"node": ["storage", "main"]
}
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])]
pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME
storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME
storage_class['parameters']['diskSelector'] = 'ssd,nvme'
storage_class['parameters']['nodeSelector'] = 'storage,main'
volume_size = DEFAULT_VOLUME_SIZE * Gi
create_storage(core_api, storage_class, pvc)
create_and_wait_pod(core_api, pod)
pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name'])
# Confirm that the volume has all the correct parameters we gave it.
volumes = client.list_volume()
assert len(volumes) == 1
assert volumes.data[0].name == pvc_volume_name
assert volumes.data[0].size == str(volume_size)
assert volumes.data[0].numberOfReplicas == \
int(storage_class['parameters']['numberOfReplicas'])
assert volumes.data[0].state == "attached"
check_volume_replicas(volumes.data[0], tag_spec, node_default_tags)
def test_tag_scheduling(client, node_default_tags): # NOQA
"""
Test that scheduling succeeds if there are available Nodes/Disks with the
requested Tags.
"""
host_id = get_self_host_id()
tag_specs = [
# Select all Nodes.
{
"disk": [],
"expected": 3,
"node": []
},
# Selector works with AND on Disk Tags.
{
"disk": ["ssd", "nvme"],
"expected": 2,
"node": []
},
# Selector works with AND on Node Tags.
{
"disk": [],
"expected": 2,
"node": ["main", "storage"]
},
# Selector works based on combined Disk AND Node selector.
{
"disk": ["ssd", "nvme"],
"expected": 1,
"node": ["storage", "main"]
}
]
for specs in tag_specs:
volume_name = generate_volume_name() # NOQA
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3,
diskSelector=specs["disk"],
nodeSelector=specs["node"])
volume = wait_for_volume_detached(client, volume_name)
assert volume["diskSelector"] == specs["disk"]
assert volume["nodeSelector"] == specs["node"]
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 3
check_volume_replicas(volume, specs, node_default_tags)
cleanup_volume(client, volume)
def test_tag_scheduling_on_update(client, node_default_tags, volume_name): # NOQA
"""
Test that Replicas get scheduled if a Node/Disk disks updated with the
proper Tags.
"""
tag_spec = {
"disk": ["ssd", "m2"],
"expected": 1,
"node": ["main", "fallback"]
}
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3,
diskSelector=tag_spec["disk"],
nodeSelector=tag_spec["node"])
volume = wait_for_volume_detached(client, volume_name)
assert volume["diskSelector"] == tag_spec["disk"]
assert volume["nodeSelector"] == tag_spec["node"]
wait_scheduling_failure(client, volume_name)
host_id = get_self_host_id()
node = client.by_id_node(host_id)
update_disks = get_update_disks(node["disks"])
update_disks[0]["tags"] = tag_spec["disk"]
node = node.diskUpdate(disks=update_disks)
set_node_tags(client, node, tag_spec["node"])
scheduled = False
for i in range(RETRY_COUNTS):
v = client.by_id_volume(volume_name)
if v["conditions"]["scheduled"]["status"] == "True":
scheduled = True
if scheduled:
break
sleep(RETRY_INTERVAL)
assert scheduled
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
nodes = client.list_node()
node_mapping = {node["id"]: {
"disk": get_update_disks(node["disks"])[0]["tags"],
"node": node["tags"]
} for node in nodes}
assert len(volume["replicas"]) == 3
check_volume_replicas(volume, tag_spec, node_mapping)
cleanup_volume(client, volume)
def test_provisioner_tags(client, core_api, node_default_tags, storage_class,
pvc, pod): # NOQA
"""
Test that a StorageClass can properly provision a volume with requested
Tags.
"""
# Prepare pod and volume specs.
pod_name = 'provisioner-tags-test'
tag_spec = {
"disk": ["ssd", "nvme"],
"expected": 1,
"node": ["storage", "main"]
}
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])]
pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME
storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME
storage_class['parameters']['diskSelector'] = 'ssd,nvme'
storage_class['parameters']['nodeSelector'] = 'storage,main'
volume_size = DEFAULT_VOLUME_SIZE * Gi
create_storage(core_api, storage_class, pvc)
create_and_wait_pod(core_api, pod)
pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name'])
# Confirm that the volume has all the correct parameters we gave it.
volumes = client.list_volume()
assert len(volumes) == 1
assert volumes[0]["name"] == pvc_volume_name
assert volumes[0]["size"] == str(volume_size)
assert volumes[0]["numberOfReplicas"] == \
int(storage_class['parameters']['numberOfReplicas'])
assert volumes[0]["state"] == "attached"
check_volume_replicas(volumes[0], tag_spec, node_default_tags)
def test_tag_scheduling(client, node_default_tags): # NOQA
"""
Test success scheduling with tags
Test prerequisites:
- set Replica Node Level Soft Anti-Affinity enabled
Case 1:
Don't specify any tags, replica should be scheduled to 3 disks.
Case 2:
Use disk tags to select two nodes for all replicas.
Case 3:
Use node tags to select two nodes for all replicas.
Case 4:
Combine node and disk tags to schedule all replicas on one node.
"""
replica_node_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(replica_node_soft_anti_affinity_setting, value="true")
host_id = get_self_host_id()
tag_specs = [
# Select all Nodes.
{
"disk": [],
"expected": 3,
"node": []
},
# Selector works with AND on Disk Tags.
{
"disk": ["ssd", "nvme"],
"expected": 2,
"node": []
},
# Selector works with AND on Node Tags.
{
"disk": [],
"expected": 2,
"node": ["main", "storage"]
},
# Selector works based on combined Disk AND Node selector.
{
"disk": ["ssd", "nvme"],
"expected": 1,
"node": ["storage", "main"]
}
]
for specs in tag_specs:
volume_name = generate_volume_name() # NOQA
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=3,
diskSelector=specs["disk"],
nodeSelector=specs["node"])
volume = wait_for_volume_detached(client, volume_name)
assert volume.diskSelector == specs["disk"]
assert volume.nodeSelector == specs["node"]
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert len(volume.replicas) == 3
check_volume_replicas(volume, specs, node_default_tags)
cleanup_volume(client, volume)
def test_tag_scheduling_on_update(client, node_default_tags,
volume_name): # NOQA
"""
Test that Replicas get scheduled if a Node/Disk disks updated with the
proper Tags.
Test prerequisites:
- set Replica Node Level Soft Anti-Affinity enabled
1. Create volume with tags that can not be satisfied
2. Wait for volume to fail scheduling
3. Update the node and disk with extra tags to satisfy the volume
4. Verify now volume has been scheduled
5. Attach the volume and check the replicas has been scheduled properly
"""
replica_node_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(replica_node_soft_anti_affinity_setting, value="true")
tag_spec = {
"disk": ["ssd", "m2"],
"expected": 1,
"node": ["main", "fallback"]
}
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=3,
diskSelector=tag_spec["disk"],
nodeSelector=tag_spec["node"])
volume = wait_for_volume_detached(client, volume_name)
assert volume.diskSelector == tag_spec["disk"]
assert volume.nodeSelector == tag_spec["node"]
wait_scheduling_failure(client, volume_name)
host_id = get_self_host_id()
node = client.by_id_node(host_id)
update_disks = get_update_disks(node.disks)
update_disks[list(update_disks)[0]].tags = tag_spec["disk"]
node = update_node_disks(client, node.name, disks=update_disks)
set_node_tags(client, node, tag_spec["node"])
scheduled = False
for i in range(RETRY_COUNTS):
v = client.by_id_volume(volume_name)
if v.conditions.scheduled.status == "True":
scheduled = True
if scheduled:
break
sleep(RETRY_INTERVAL)
assert scheduled
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
nodes = client.list_node()
node_mapping = {
node.id: {
"disk": node.disks[list(node.disks)[0]].tags,
"node": node.tags
}
for node in nodes
}
assert len(volume.replicas) == 3
check_volume_replicas(volume, tag_spec, node_mapping)
cleanup_volume(client, volume)