def check_workload_update(core_api, apps_api, count): # NOQA
da_list = apps_api.list_namespaced_daemon_set(LONGHORN_NAMESPACE).items
for da in da_list:
if da.status.updated_number_scheduled != count:
return False
dp_list = apps_api.list_namespaced_deployment(LONGHORN_NAMESPACE).items
for dp in dp_list:
if dp.status.updated_replicas != dp.spec.replicas:
return False
im_pod_list = core_api.list_namespaced_pod(
LONGHORN_NAMESPACE,
label_selector="longhorn.io/component=instance-manager").items
if len(im_pod_list) != 2 * count:
return False
for p in im_pod_list:
if p.status.phase != "Running":
return False
client = get_longhorn_api_client() # NOQA
images = client.list_engine_image()
assert len(images) == 1
ei_state = get_engine_image_status_value(client, images[0].name)
if images[0].state != ei_state:
return False
return True
def test_engine_image_incompatible(client, volume_name): # NOQA
images = client.list_engine_image()
assert len(images) == 1
assert images[0]["default"]
assert images[0]["state"] == "ready"
cli_v = images[0]["cliAPIVersion"]
# cli_minv = images[0]["cliAPIMinVersion"]
ctl_v = images[0]["controllerAPIVersion"]
ctl_minv = images[0]["controllerAPIMinVersion"]
data_v = images[0]["dataFormatVersion"]
data_minv = images[0]["dataFormatMinVersion"]
fail_cli_v_image = common.get_compatibility_test_image(
cli_v - 1, cli_v - 1, ctl_v, ctl_minv, data_v, data_minv)
img = client.create_engine_image(image=fail_cli_v_image)
img_name = img["name"]
img = wait_for_engine_image_state(client, img_name, "incompatible")
assert img["state"] == "incompatible"
assert img["cliAPIVersion"] == cli_v - 1
assert img["cliAPIMinVersion"] == cli_v - 1
client.delete(img)
fail_cli_minv_image = common.get_compatibility_test_image(
cli_v + 1, cli_v + 1, ctl_v, ctl_minv, data_v, data_minv)
img = client.create_engine_image(image=fail_cli_minv_image)
img_name = img["name"]
img = wait_for_engine_image_state(client, img_name, "incompatible")
assert img["state"] == "incompatible"
assert img["cliAPIVersion"] == cli_v + 1
assert img["cliAPIMinVersion"] == cli_v + 1
client.delete(img)
def wait_for_toleration_update(core_api, apps_api, count,
set_tolerations): # NOQA
updated = False
for i in range(RETRY_COUNTS):
time.sleep(RETRY_INTERVAL_LONG)
updated = True
da_list = apps_api.list_namespaced_daemon_set(LONGHORN_NAMESPACE).items
for da in da_list:
if da.status.updated_number_scheduled != count:
updated = False
break
if not updated:
continue
dp_list = apps_api.list_namespaced_deployment(LONGHORN_NAMESPACE).items
for dp in dp_list:
if dp.status.updated_replicas != dp.spec.replicas:
updated = False
break
if not updated:
continue
im_pod_list = core_api.list_namespaced_pod(
LONGHORN_NAMESPACE,
label_selector="longhorn.io/component=instance-manager").items
if len(im_pod_list) != 2 * count:
updated = False
continue
for p in im_pod_list:
if p.status.phase != "Running":
updated = False
break
if not updated:
continue
pod_list = core_api.list_namespaced_pod(LONGHORN_NAMESPACE).items
for p in pod_list:
if p.status.phase != "Running" or \
not check_tolerations_set(p.spec.tolerations,
set_tolerations):
updated = False
break
if not updated:
continue
client = get_longhorn_api_client() # NOQA
images = client.list_engine_image()
assert len(images) == 1
if images[0].state != "ready":
updated = False
continue
if updated:
break
assert updated
def test_engine_image(client, volume_name): # NOQA
# can be leftover
default_img = common.get_default_engine_image(client)
default_img_name = default_img["name"]
default_img = wait_for_engine_image_ref_count(client, default_img_name, 0)
images = client.list_engine_image()
assert len(images) == 1
assert images[0]["default"]
assert images[0]["state"] == "ready"
assert images[0]["refCount"] == 0
assert images[0]["gitCommit"] != ""
assert images[0]["buildDate"] != ""
cli_v = default_img["cliAPIVersion"]
cli_minv = default_img["cliAPIMinVersion"]
ctl_v = default_img["controllerAPIVersion"]
ctl_minv = default_img["controllerAPIMinVersion"]
data_v = default_img["dataFormatVersion"]
data_minv = default_img["dataFormatMinVersion"]
assert cli_v != 0
assert cli_minv != 0
assert ctl_v != 0
assert ctl_minv != 0
assert data_v != 0
assert data_minv != 0
# delete default image is not allowed
with pytest.raises(Exception) as e:
client.delete(images[0])
assert "the default engine image" in str(e.value)
# duplicate images
with pytest.raises(Exception) as e:
client.create_engine_image(image=default_img["image"])
engine_upgrade_image = common.get_upgrade_test_image(cli_v, cli_minv,
ctl_v, ctl_minv,
data_v, data_minv)
new_img = client.create_engine_image(image=engine_upgrade_image)
new_img_name = new_img["name"]
new_img = wait_for_engine_image_state(client, new_img_name, "ready")
assert not new_img["default"]
assert new_img["state"] == "ready"
assert new_img["refCount"] == 0
assert new_img["cliAPIVersion"] != 0
assert new_img["cliAPIMinVersion"] != 0
assert new_img["controllerAPIVersion"] != 0
assert new_img["controllerAPIMinVersion"] != 0
assert new_img["dataFormatVersion"] != 0
assert new_img["dataFormatMinVersion"] != 0
assert new_img["gitCommit"] != ""
assert new_img["buildDate"] != ""
client.delete(new_img)
def test_engine_image_incompatible(client, core_api, volume_name): # NOQA
"""
Test incompatible engine images
1. Deploy incompatible engine images
2. Make sure their state are `incompatible` once deployed.
"""
images = client.list_engine_image()
assert len(images) == 1
assert images[0].default
assert images[0].state == "ready"
cli_v = images[0].cliAPIVersion
# cli_minv = images[0].cliAPIMinVersion
ctl_v = images[0].controllerAPIVersion
ctl_minv = images[0].controllerAPIMinVersion
data_v = images[0].dataFormatVersion
data_minv = images[0].dataFormatMinVersion
fail_cli_v_image = common.get_compatibility_test_image(
cli_v - 1, cli_v - 1, ctl_v, ctl_minv, data_v, data_minv)
img = client.create_engine_image(image=fail_cli_v_image)
img_name = img.name
img = wait_for_engine_image_state(client, img_name, "incompatible")
assert img.state == "incompatible"
assert img.cliAPIVersion == cli_v - 1
assert img.cliAPIMinVersion == cli_v - 1
client.delete(img)
wait_for_engine_image_deletion(client, core_api, img.name)
fail_cli_minv_image = common.get_compatibility_test_image(
cli_v + 1, cli_v + 1, ctl_v, ctl_minv, data_v, data_minv)
img = client.create_engine_image(image=fail_cli_minv_image)
img_name = img.name
img = wait_for_engine_image_state(client, img_name, "incompatible")
assert img.state == "incompatible"
assert img.cliAPIVersion == cli_v + 1
assert img.cliAPIMinVersion == cli_v + 1
client.delete(img)
wait_for_engine_image_deletion(client, core_api, img.name)
def test_upgrade(
upgrade_longhorn_manager_repo_url,
upgrade_longhorn_manager_repo_branch,
upgrade_longhorn_manager_image,
upgrade_longhorn_engine_image,
upgrade_longhorn_instance_manager_image,
upgrade_longhorn_share_manager_image,
upgrade_longhorn_backing_image_manager_image,
client,
core_api,
volume_name,
csi_pv, # NOQA
pvc,
pod_make,
statefulset,
storage_class): # NOQA
"""
Test Longhorn upgrade
TODO
The test will cover both volume has revision counter enabled and
disabled cases.
Prerequisite:
- Disable Auto Salvage Setting
1. Find the upgrade image tag
2. Create a volume, generate and write data into the volume.
1. Create a volume with revision counter enabled case.
2. Create a volume with revision counter disabled case.
3. Create a Pod using a volume, generate and write data
4. Create a StatefulSet with 2 replicas,
generate and write data to their volumes
5. Keep all volumes attached
6. Upgrade Longhorn system.
7. Check Pod and StatefulSet didn't restart after upgrade
8. Check All volumes data
9. Write data to StatefulSet pods, and Attached volume
10. Check data written to StatefulSet pods, and attached volume.
11. Detach the volume, and Delete Pod, and
StatefulSet to detach theirvolumes
12. Upgrade all volumes engine images.
13. Attach the volume, and recreate Pod, and StatefulSet
14. Check All volumes data
"""
longhorn_manager_repo = upgrade_longhorn_manager_repo_url
longhorn_manager_branch = upgrade_longhorn_manager_repo_branch
longhorn_manager_image = upgrade_longhorn_manager_image
longhorn_engine_image = upgrade_longhorn_engine_image
longhorn_instance_manager_image = upgrade_longhorn_instance_manager_image
longhorn_share_manager_image = upgrade_longhorn_share_manager_image
longhorn_backing_image_manager_image = \
upgrade_longhorn_backing_image_manager_image
host_id = get_self_host_id()
pod_data_path = "/data/test"
pod_volume_name = generate_volume_name()
auto_salvage_setting = client.by_id_setting(SETTING_AUTO_SALVAGE)
setting = client.update(auto_salvage_setting, value="false")
assert setting.name == SETTING_AUTO_SALVAGE
assert setting.value == "false"
# Create Volume attached to a node.
volume1 = create_and_check_volume(client, volume_name, size=SIZE)
volume1.attach(hostId=host_id)
volume1 = wait_for_volume_healthy(client, volume_name)
volume1_data = write_volume_random_data(volume1)
# Create Volume used by Pod
pod_name, pv_name, pvc_name, pod_md5sum = \
prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc,
pod_make, pod_volume_name,
data_path=pod_data_path,
add_liveness_probe=False)
# Create multiple volumes used by StatefulSet
statefulset_name = 'statefulset-upgrade-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
statefulset_pod_info = get_statefulset_pod_info(core_api, statefulset)
for sspod_info in statefulset_pod_info:
sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, sspod_info['pod_name'],
sspod_info['data'])
# upgrade Longhorn
assert longhorn_upgrade(longhorn_manager_repo, longhorn_manager_branch,
longhorn_manager_image, longhorn_engine_image,
longhorn_instance_manager_image,
longhorn_share_manager_image,
longhorn_backing_image_manager_image)
client = get_longhorn_api_client()
# wait for 1 minute before checking pod restarts
time.sleep(60)
pod = core_api.read_namespaced_pod(name=pod_name, namespace='default')
assert pod.status.container_statuses[0].restart_count == 0
for sspod_info in statefulset_pod_info:
sspod = core_api.read_namespaced_pod(name=sspod_info['pod_name'],
namespace='default')
assert \
sspod.status.container_statuses[0].restart_count == 0
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
assert res_pod_md5sum == pod_md5sum
check_volume_data(volume1, volume1_data)
for sspod_info in statefulset_pod_info:
sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, sspod_info['pod_name'],
sspod_info['data'])
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
volume1 = client.by_id_volume(volume_name)
volume1_data = write_volume_random_data(volume1)
check_volume_data(volume1, volume1_data)
statefulset['spec']['replicas'] = replicas = 0
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={'spec': {
'replicas': replicas
}})
delete_and_wait_pod(core_api, pod_name)
volume = client.by_id_volume(volume_name)
volume.detach(hostId="")
volumes = client.list_volume()
for v in volumes:
wait_for_volume_detached(client, v.name)
engineimages = client.list_engine_image()
for ei in engineimages:
if ei.image == longhorn_engine_image:
new_ei = ei
volumes = client.list_volume()
for v in volumes:
volume = client.by_id_volume(v.name)
volume.engineUpgrade(image=new_ei.image)
statefulset['spec']['replicas'] = replicas = 2
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={'spec': {
'replicas': replicas
}})
wait_statefulset(statefulset)
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
create_and_wait_pod(core_api, pod)
volume1 = client.by_id_volume(volume_name)
volume1.attach(hostId=host_id)
volume1 = wait_for_volume_healthy(client, volume_name)
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
assert res_pod_md5sum == pod_md5sum
check_volume_data(volume1, volume1_data)
def test_engine_image(client, core_api, volume_name): # NOQA
"""
Test Engine Image deployment
1. List Engine Images and validate basic properities.
2. Try deleting default engine image and it should fail.
3. Try creating a duplicate engine image as default and it should fail
4. Get upgrade test image for the same versions
5. Test if the upgrade test image can be deployed and deleted correctly
"""
# can be leftover
default_img = common.get_default_engine_image(client)
default_img_name = default_img.name
default_img = wait_for_engine_image_ref_count(client, default_img_name, 0)
ei_state = get_engine_image_status_value(client, default_img_name)
images = client.list_engine_image()
assert len(images) == 1
assert images[0].default
assert images[0].state == ei_state
assert images[0].refCount == 0
assert images[0].gitCommit != ""
assert images[0].buildDate != ""
cli_v = default_img.cliAPIVersion
cli_minv = default_img.cliAPIMinVersion
ctl_v = default_img.controllerAPIVersion
ctl_minv = default_img.controllerAPIMinVersion
data_v = default_img.dataFormatVersion
data_minv = default_img.dataFormatMinVersion
assert cli_v != 0
assert cli_minv != 0
assert ctl_v != 0
assert ctl_minv != 0
assert data_v != 0
assert data_minv != 0
# delete default image is not allowed
with pytest.raises(Exception) as e:
client.delete(images[0])
assert "the default engine image" in str(e.value)
# duplicate images
with pytest.raises(Exception) as e:
client.create_engine_image(image=default_img.image)
engine_upgrade_image = common.get_upgrade_test_image(
cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv)
# test if engine image can be created and cleaned up successfully
for _ in range(ENGINE_IMAGE_TEST_REPEAT_COUNT):
new_img = client.create_engine_image(image=engine_upgrade_image)
new_img_name = new_img.name
new_img = wait_for_engine_image_state(client, new_img_name, ei_state)
assert not new_img.default
assert new_img.state == ei_state
assert new_img.refCount == 0
assert new_img.cliAPIVersion != 0
assert new_img.cliAPIMinVersion != 0
assert new_img.controllerAPIVersion != 0
assert new_img.controllerAPIMinVersion != 0
assert new_img.dataFormatVersion != 0
assert new_img.dataFormatMinVersion != 0
assert new_img.gitCommit != ""
assert new_img.buildDate != ""
client.delete(new_img)
wait_for_engine_image_deletion(client, core_api, new_img.name)