def test_csi_snapshot_with_invalid_param(
volumesnapshotclass, # NOQA
volumesnapshot, # NOQA
client, # NOQA
core_api, # NOQA
volume_name, # NOQA
csi_pv, # NOQA
pvc, # NOQA
pod_make, # NOQA
request): # NOQA
"""
Context:
After deploy the CSI snapshot CRDs, Controller at
https://longhorn.io/docs/1.2.4/snapshots-and-backups/
csi-snapshot-support/enable-csi-snapshot-support/
Create VolumeSnapshotClass with type=invalid
- invalid (type=invalid)
Test the extend CSI snapshot type=invalid behavior to Longhorn snapshot
Steps:
0. Create Longhorn volume test-vol
- Size 5GB
- Create PV/PVC for the Longhorn volume
- Write data into volume
- Setup backup store
1. Test create CSI snapshot
- Create VolumeSnapshot with class invalid
- Verify that the volumesnapshot object is not ready
"""
# Step 0
csi_snapshot_type = "invalid"
csisnapclass = \
volumesnapshotclass(name="snapshotclass-invalid",
deletepolicy="Delete",
snapshot_type=csi_snapshot_type)
pod_name, pv_name, pvc_name, md5sum = \
prepare_pod_with_data_in_mb(client, core_api,
csi_pv, pvc, pod_make,
volume_name,
data_path="/data/test")
# Step 1
csivolsnap = volumesnapshot(volume_name + "-volumesnapshot", "default",
csisnapclass["metadata"]["name"],
"persistentVolumeClaimName", pvc_name)
wait_for_volumesnapshot_ready(
volumesnapshot_name=csivolsnap["metadata"]["name"],
namespace='default',
ready_to_use=False)
def finalizer():
delete_volumesnapshot(csivolsnap["metadata"]["name"], 'default')
request.addfinalizer(finalizer)
def test_restore_rwo_volume_to_rwx(set_random_backupstore, client, core_api,
volume_name, pvc, csi_pv, pod_make,
make_deployment_with_pvc): # NOQA
"""
Test restoring a rwo to a rwx volume.
1. Create a volume with 'accessMode' rwo.
2. Create a PV and a PVC with access mode 'readwriteonce' and attach to the
volume.
3. Create a pod and attach to the PVC.
4. Write some data into the pod and compute md5sum.
5. Take a backup of the volume.
6. Restore the backup with 'accessMode' rwx.
7. Create PV and PVC and attach to 2 pods.
8. Verify the data.
"""
data_path = "/data/test"
pod_name, pv_name, pvc_name, md5sum = \
prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc,
pod_make,
volume_name,
data_size_in_mb=DATA_SIZE_IN_MB_1,
data_path=data_path)
snap = create_snapshot(client, volume_name)
volume = client.by_id_volume(volume_name)
volume.snapshotBackup(name=snap.name)
wait_for_backup_completion(client, volume_name, snap.name)
bv, b1 = find_backup(client, volume_name, snap.name)
restore_volume_name = 'restored-rwx-volume'
restore_pv_name = restore_volume_name + "-pv"
restore_pvc_name = restore_volume_name + "-pvc"
client.create_volume(name=restore_volume_name,
size=str(1 * Gi),
numberOfReplicas=3,
fromBackup=b1.url,
accessMode='rwx')
wait_for_volume_creation(client, restore_volume_name)
restore_volume = wait_for_volume_detached(client, restore_volume_name)
create_pv_for_volume(client, core_api, restore_volume, restore_pv_name)
create_pvc_for_volume(client, core_api, restore_volume, restore_pvc_name)
deployment = make_deployment_with_pvc('deployment-multi-pods-test',
restore_pvc_name,
replicas=2)
apps_api = get_apps_api_client()
create_and_wait_deployment(apps_api, deployment)
deployment_label_selector = \
"name=" + deployment["metadata"]["labels"]["name"]
deployment_pod_list = \
core_api.list_namespaced_pod(namespace="default",
label_selector=deployment_label_selector)
pod_name_1 = deployment_pod_list.items[0].metadata.name
pod_name_2 = deployment_pod_list.items[1].metadata.name
md5sum_pod1 = get_pod_data_md5sum(core_api, pod_name_1, data_path)
md5sum_pod2 = get_pod_data_md5sum(core_api, pod_name_2, data_path)
assert md5sum == md5sum_pod1 == md5sum_pod2
def test_csi_volumesnapshot_restore_existing_backup(
set_random_backupstore, # NOQA
client, # NOQA
core_api, # NOQA
volume_name, # NOQA
csi_pv, # NOQA
pvc, # NOQA
pod_make, # NOQA
volumesnapshotclass, # NOQA
volumesnapshotcontent,
volumesnapshot, # NOQA
volsnapshotclass_delete_policy, # NOQA
backup_is_deleted): # NOQA
"""
Test retention of a backup while deleting the associated `VolumeSnapshot`
via the csi snapshotter
Context:
We want to allow the user to programmatically create/restore/delete
longhorn backups via the csi snapshot mechanism
ref: https://kubernetes.io/docs/concepts/storage/volume-snapshots/
Setup:
1. Make sure your cluster contains the below crds
https://github.com/kubernetes-csi/external-snapshotter
/tree/master/client/config/crd
2. Make sure your cluster contains the snapshot controller
https://github.com/kubernetes-csi/external-snapshotter
/tree/master/deploy/kubernetes/snapshot-controller
Steps:
1. create new snapshotClass with deletionPolicy set to Retain
2. call csi_volumesnapshot_creation_test(snapshotClass=custom)
3. call csi_volumesnapshot_restore_test()
4. call csi_volumesnapshot_deletion_test(deletionPolicy='Retain'):
5. cleanup
"""
csisnapclass = \
volumesnapshotclass(name="snapshotclass",
deletepolicy=volsnapshotclass_delete_policy)
pod_name, pv_name, pvc_name, md5sum = \
prepare_pod_with_data_in_mb(client, core_api,
csi_pv, pvc, pod_make,
volume_name,
data_path="/data/test")
volume = client.by_id_volume(volume_name)
snap = create_snapshot(client, volume_name)
volume.snapshotBackup(name=snap.name)
wait_for_backup_completion(client, volume_name, snap.name)
bv, b = find_backup(client, volume_name, snap.name)
csivolsnap_name = volume_name + "-volumesnapshot"
csivolsnap_namespace = "default"
volsnapcontent = \
volumesnapshotcontent("volsnapcontent",
csisnapclass["metadata"]["name"],
"Delete",
"bs://" + volume_name + "/" + b.name,
csivolsnap_name,
csivolsnap_namespace)
csivolsnap = volumesnapshot(csivolsnap_name, csivolsnap_namespace,
csisnapclass["metadata"]["name"],
"volumeSnapshotContentName",
volsnapcontent["metadata"]["name"])
restore_pvc_name = pvc["metadata"]["name"] + "-restore"
restore_pvc_size = pvc["spec"]["resources"]["requests"]["storage"]
restore_csi_volume_snapshot(core_api, client, csivolsnap, restore_pvc_name,
restore_pvc_size)
restore_pod = pod_make()
restore_pod_name = restore_pod["metadata"]["name"]
restore_pod['spec']['volumes'] = [create_pvc_spec(restore_pvc_name)]
create_and_wait_pod(core_api, restore_pod)
restore_md5sum = \
get_pod_data_md5sum(core_api, restore_pod_name, path="/data/test")
assert restore_md5sum == md5sum
# Delete volumeSnapshot test
delete_volumesnapshot(csivolsnap["metadata"]["name"], "default")
if backup_is_deleted is False:
find_backup(client, volume_name, b["snapshotName"])
else:
wait_for_backup_delete(client, volume_name, b["name"])
def test_csi_volumesnapshot_basic(
set_random_backupstore, # NOQA
volumesnapshotclass, # NOQA
volumesnapshot, # NOQA
client, # NOQA
core_api, # NOQA
volume_name, # NOQA
csi_pv, # NOQA
pvc, # NOQA
pod_make, # NOQA
volsnapshotclass_delete_policy, # NOQA
backup_is_deleted): # NOQA
"""
Test creation / restoration / deletion of a backup via the csi snapshotter
Context:
We want to allow the user to programmatically create/restore/delete
longhorn backups via the csi snapshot mechanism
ref: https://kubernetes.io/docs/concepts/storage/volume-snapshots/
Setup:
1. Make sure your cluster contains the below crds
https://github.com/kubernetes-csi/external-snapshotter
/tree/master/client/config/crd
2. Make sure your cluster contains the snapshot controller
https://github.com/kubernetes-csi/external-snapshotter
/tree/master/deploy/kubernetes/snapshot-controller
Steps:
def csi_volumesnapshot_creation_test(snapshotClass=longhorn|custom):
1. create volume(1)
2. write data to volume(1)
3. create a kubernetes `VolumeSnapshot` object
the `VolumeSnapshot.uuid` will be used to identify a
**longhorn snapshot** and the associated `VolumeSnapshotContent` object
4. check creation of a new longhorn snapshot named `snapshot-uuid`
5. check for `VolumeSnapshotContent` named `snapcontent-uuid`
6. wait for `VolumeSnapshotContent.readyToUse` flag to be set to **true**
7. check for backup existance on the backupstore
# the csi snapshot restore sets the fromBackup field same as
# the StorageClass based restore approach.
def csi_volumesnapshot_restore_test():
8. create a `PersistentVolumeClaim` object where the `dataSource` field
references the `VolumeSnapshot` object by name
9. verify creation of a new volume(2) bound to the pvc created in step(8)
10. verify data of new volume(2) equals data
from backup (ie old data above)
# default longhorn snapshot class is set to Delete
# add a second test with a custom snapshot class with deletionPolicy
# set to Retain you can reuse these methods for that and other tests
def csi_volumesnapshot_deletion_test(deletionPolicy='Delete|Retain'):
11. delete `VolumeSnapshot` object
12. if deletionPolicy == Delete:
13. verify deletion of `VolumeSnapshot` and
`VolumeSnapshotContent` objects
14. verify deletion of backup from backupstore
12. if deletionPolicy == Retain:
13. verify deletion of `VolumeSnapshot`
14. verify retention of `VolumeSnapshotContent`
and backup on backupstore
15. cleanup
"""
csisnapclass = \
volumesnapshotclass(name="snapshotclass",
deletepolicy=volsnapshotclass_delete_policy)
pod_name, pv_name, pvc_name, md5sum = \
prepare_pod_with_data_in_mb(client, core_api,
csi_pv, pvc, pod_make,
volume_name,
data_path="/data/test")
# Create volumeSnapshot test
csivolsnap = volumesnapshot(volume_name + "-volumesnapshot", "default",
csisnapclass["metadata"]["name"],
"persistentVolumeClaimName", pvc_name)
volume = client.by_id_volume(volume_name)
for i in range(RETRY_COUNTS):
snapshots = volume.snapshotList()
if len(snapshots) == 2:
break
time.sleep(RETRY_INTERVAL)
lh_snapshot = None
snapshots = volume.snapshotList()
for snapshot in snapshots:
if snapshot["name"] == "snapshot-" + csivolsnap["metadata"]["uid"]:
lh_snapshot = snapshot
assert lh_snapshot is not None
wait_for_volumesnapshot_ready(csivolsnap["metadata"]["name"],
csivolsnap["metadata"]["namespace"])
bv1, b = find_backup(client, volume_name, lh_snapshot["name"])
assert b["snapshotName"] == lh_snapshot["name"]
restore_pvc_name = pvc["metadata"]["name"] + "-restore"
restore_pvc_size = pvc["spec"]["resources"]["requests"]["storage"]
restore_csi_volume_snapshot(core_api, client, csivolsnap, restore_pvc_name,
restore_pvc_size)
restore_pod = pod_make()
restore_pod_name = restore_pod["metadata"]["name"]
restore_pod['spec']['volumes'] = [create_pvc_spec(restore_pvc_name)]
create_and_wait_pod(core_api, restore_pod)
restore_md5sum = \
get_pod_data_md5sum(core_api, restore_pod_name, path="/data/test")
assert restore_md5sum == md5sum
# Delete volumeSnapshot test
delete_volumesnapshot(csivolsnap["metadata"]["name"], "default")
if backup_is_deleted is False:
find_backup(client, volume_name, b["snapshotName"])
else:
wait_for_backup_delete(client, volume_name, b["name"])
def test_upgrade(upgrade_image_tag, settings_reset, volume_name, pod_make, statefulset, storage_class): # NOQA
"""
Test Longhorn upgrade
Prerequisite:
- Disable Auto Salvage Setting
1. Find the upgrade image tag
2. Create a volume, generate and write data into the volume.
3. Create a Pod using a volume, generate and write data
4. Create a StatefulSet with 2 replicas,
generate and write data to their volumes
5. Keep all volumes attached
6. Upgrade Longhorn system.
7. Check Pod and StatefulSet didn't restart after upgrade
8. Check All volumes data
9. Write data to StatefulSet pods, and Attached volume
10. Check data written to StatefulSet pods, and attached volume.
11. Detach the volume, and Delete Pod, and
StatefulSet to detach theirvolumes
12. Upgrade all volumes engine images.
13. Attach the volume, and recreate Pod, and StatefulSet
14. Check All volumes data
"""
new_ei_name = "longhornio/longhorn-engine:" + upgrade_image_tag
client = get_longhorn_api_client()
core_api = get_core_api_client()
host_id = get_self_host_id()
pod_data_path = "/data/test"
pod_volume_name = generate_volume_name()
auto_salvage_setting = client.by_id_setting(SETTING_AUTO_SALVAGE)
setting = client.update(auto_salvage_setting, value="false")
assert setting.name == SETTING_AUTO_SALVAGE
assert setting.value == "false"
# Create Volume attached to a node.
volume1 = create_and_check_volume(client,
volume_name,
size=SIZE)
volume1.attach(hostId=host_id)
volume1 = wait_for_volume_healthy(client, volume_name)
volume1_data = write_volume_random_data(volume1)
# Create Volume used by Pod
pod_name, pv_name, pvc_name, pod_md5sum = \
prepare_pod_with_data_in_mb(client, core_api,
pod_make, pod_volume_name,
data_path=pod_data_path,
add_liveness_prope=False)
# Create multiple volumes used by StatefulSet
statefulset_name = 'statefulset-upgrade-test'
update_statefulset_manifests(statefulset,
storage_class,
statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
statefulset_pod_info = get_statefulset_pod_info(core_api, statefulset)
for sspod_info in statefulset_pod_info:
sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api,
sspod_info['pod_name'],
sspod_info['data'])
# upgrade Longhorn
assert longhorn_upgrade(upgrade_image_tag)
client = get_longhorn_api_client()
# wait for 1 minute before checking pod restarts
time.sleep(60)
pod = core_api.read_namespaced_pod(name=pod_name,
namespace='default')
assert pod.status.container_statuses[0].restart_count == 0
for sspod_info in statefulset_pod_info:
sspod = core_api.read_namespaced_pod(name=sspod_info['pod_name'],
namespace='default')
assert \
sspod.status.container_statuses[0].restart_count == 0
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
assert res_pod_md5sum == pod_md5sum
check_volume_data(volume1, volume1_data)
for sspod_info in statefulset_pod_info:
sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api,
sspod_info['pod_name'],
sspod_info['data'])
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
volume1 = client.by_id_volume(volume_name)
volume1_data = write_volume_random_data(volume1)
check_volume_data(volume1, volume1_data)
statefulset['spec']['replicas'] = replicas = 0
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={
'spec': {
'replicas': replicas
}
})
delete_and_wait_pod(core_api, pod_name)
volume = client.by_id_volume(volume_name)
volume.detach()
volumes = client.list_volume()
for v in volumes:
wait_for_volume_detached(client, v.name)
engineimages = client.list_engine_image()
for ei in engineimages:
if ei.image == new_ei_name:
new_ei = ei
volumes = client.list_volume()
for v in volumes:
volume = client.by_id_volume(v.name)
volume.engineUpgrade(image=new_ei.image)
statefulset['spec']['replicas'] = replicas = 2
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={
'spec': {
'replicas': replicas
}
})
wait_statefulset(statefulset)
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
create_and_wait_pod(core_api, pod)
volume1 = client.by_id_volume(volume_name)
volume1.attach(hostId=host_id)
volume1 = wait_for_volume_healthy(client, volume_name)
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
assert res_pod_md5sum == pod_md5sum
check_volume_data(volume1, volume1_data)
