def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA
vol_name = csi_pv['metadata']['name']
write_pod_volume_data(core_api, pod_name, test_data)
volume = client.by_id_volume(vol_name)
snap = create_snapshot(client, vol_name)
volume.snapshotBackup(name=snap.name)
common.wait_for_backup_completion(client, vol_name, snap.name)
bv, b = common.find_backup(client, vol_name, snap.name)
pod2_name = 'csi-backup-test-' + str(i)
create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make,
base_image, b.url)
resp = read_volume_data(core_api, pod2_name)
assert resp == test_data
delete_backup(client, bv.name, b.name)
def backup_create_and_record_md5sum(client, core_api, volume_name, pod_name, snapshots_md5sum): # NOQA
volume = client.by_id_volume(volume_name)
data_md5sum = read_data_md5sum(core_api, pod_name)
snap_name = snapshot_create_and_record_md5sum(client,
core_api,
volume_name,
pod_name,
snapshots_md5sum)
snap = snapshot_data(snap_name)
snapshots_md5sum[snap_name] = snap
volume.snapshotBackup(name=snap_name)
global WAIT_BACKUP_COMPLETE
if WAIT_BACKUP_COMPLETE is None:
WAIT_BACKUP_COMPLETE = bool(random.getrandbits(1))
if WAIT_BACKUP_COMPLETE is True:
# wait for volume backupStatus to be updated with new backup record
for i in range(RETRY_COUNTS):
volume = client.by_id_volume(volume_name)
for b in volume.backupStatus:
if b.snapshot == snap_name:
if b.error == "":
wait_for_backup_completion(client,
volume_name,
snap_name)
break
else:
print("...aborting backup " + b.error)
return
time.sleep(RETRY_INTERVAL)
_, b = find_backup(client, volume_name, snap_name)
snap.set_backup_name(b["name"])
snap.set_backup_url(b["url"])
snap.set_data_md5sum(data_md5sum)
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, vol_name, backing_image, test_data): # NOQA
write_pod_volume_data(core_api, pod_name, test_data)
volume = client.by_id_volume(vol_name)
snap = create_snapshot(client, vol_name)
volume.snapshotBackup(name=snap.name)
common.wait_for_backup_completion(client, vol_name, snap.name)
bv, b = common.find_backup(client, vol_name, snap.name)
pod2_name = 'csi-backup-test-2'
vol2_name = create_and_wait_csi_pod(
pod2_name, client, core_api, csi_pv, pvc, pod_make,
backing_image, b.url)
volume2 = client.by_id_volume(vol2_name)
resp = read_volume_data(core_api, pod2_name)
assert resp == test_data
delete_backup(client, bv.name, b.name)
delete_and_wait_pod(core_api, pod2_name)
client.delete(volume2)
def test_backup_kubernetes_status(set_random_backupstore, client, core_api,
pod): # NOQA
"""
Test that Backups have KubernetesStatus stored properly when there is an
associated PersistentVolumeClaim and Pod.
1. Setup a random backupstore
2. Set settings Longhorn Static StorageClass to `longhorn-static-test`
3. Create a volume and PV/PVC. Verify the StorageClass of PVC
4. Create a Pod using the PVC.
5. Check volume's Kubernetes status to reflect PV/PVC/Pod correctly.
6. Create a backup for the volume.
7. Verify the labels of created backup reflect PV/PVC/Pod status.
8. Restore the backup to a volume. Wait for restoration to complete.
9. Check the volume's Kubernetes Status
1. Make sure the `lastPodRefAt` and `lastPVCRefAt` is snapshot created
time
10. Delete the backup and restored volume.
11. Delete PV/PVC/Pod.
12. Verify volume's Kubernetes Status updated to reflect history data.
13. Attach the volume and create another backup. Verify the labels
14. Verify the volume's Kubernetes status.
15. Restore the previous backup to a new volume. Wait for restoration.
16. Verify the restored volume's Kubernetes status.
1. Make sure `lastPodRefAt` and `lastPVCRefAt` matched volume on step
12
"""
host_id = get_self_host_id()
static_sc_name = "longhorn-static-test"
setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC)
setting = client.update(setting, value=static_sc_name)
assert setting.value == static_sc_name
volume_name = "test-backup-kubernetes-status-pod" # NOQA
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2)
volume = wait_for_volume_detached(client, volume_name)
pod_name = "pod-" + volume_name
pv_name = "pv-" + volume_name
pvc_name = "pvc-" + volume_name
create_pv_for_volume(client, core_api, volume, pv_name)
create_pvc_for_volume(client, core_api, volume, pvc_name)
ret = core_api.list_namespaced_persistent_volume_claim(namespace='default')
pvc_found = False
for item in ret.items:
if item.metadata.name == pvc_name:
pvc_found = item
break
assert pvc_found
assert pvc_found.spec.storage_class_name == static_sc_name
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pvc_name,
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'lastPodRefAt':
'',
'lastPVCRefAt':
'',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
pv_name,
'pvStatus':
'Bound',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_healthy(client, volume_name)
# Create Backup manually instead of calling create_backup since Kubernetes
# is not guaranteed to mount our Volume to the test host.
snap = create_snapshot(client, volume_name)
volume.snapshotBackup(name=snap.name)
wait_for_backup_completion(client, volume_name, snap.name)
_, b = find_backup(client, volume_name, snap.name)
# Check backup label
status = loads(b.labels.get(KUBERNETES_STATUS_LABEL))
assert status == ks
# Check backup volume label
for _ in range(RETRY_COUNTS):
bv = client.by_id_backupVolume(volume_name)
if bv is not None and bv.labels is not None:
break
time.sleep(RETRY_INTERVAL)
assert bv is not None and bv.labels is not None
status = loads(bv.labels.get(KUBERNETES_STATUS_LABEL))
assert status == ks
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b.url)
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
snapshot_created = b.snapshotCreated
ks = {
'lastPodRefAt':
b.snapshotCreated,
'lastPVCRefAt':
b.snapshotCreated,
'namespace':
'default',
'pvcName':
pvc_name,
# Restoration should not apply PersistentVolume data.
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
# We need to compare LastPodRefAt and LastPVCRefAt manually since
# wait_volume_kubernetes_status only checks for empty or non-empty state.
assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"]
assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"]
delete_backup(client, bv.name, b.name)
client.delete(restore)
wait_for_volume_delete(client, restore_name)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, pvc_name)
delete_and_wait_pv(core_api, pv_name)
# With the Pod, PVC, and PV deleted, the Volume should have both Ref
# fields set. Check that a new Backup and Restore will use this instead of
# manually populating the Ref fields.
ks = {
'lastPodRefAt':
'NOT NULL',
'lastPVCRefAt':
'NOT NULL',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
snap = create_snapshot(client, volume_name)
volume.snapshotBackup(name=snap.name)
volume = wait_for_backup_completion(client, volume_name, snap.name)
bv, b = find_backup(client, volume_name, snap.name)
new_b = bv.backupGet(name=b.name)
status = loads(new_b.labels.get(KUBERNETES_STATUS_LABEL))
# Check each field manually, we have no idea what the LastPodRefAt or the
# LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated.
assert status['lastPodRefAt'] != snapshot_created
assert status['lastPVCRefAt'] != snapshot_created
assert status['namespace'] == "default"
assert status['pvcName'] == pvc_name
assert status['pvName'] == ""
assert status['pvStatus'] == ""
assert status['workloadsStatus'] == [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b.url)
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
ks = {
'lastPodRefAt':
status['lastPodRefAt'],
'lastPVCRefAt':
status['lastPVCRefAt'],
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"]
assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"]
# cleanup
backupstore_cleanup(client)
client.delete(restore)
cleanup_volume(client, volume)
def test_restore_rwo_volume_to_rwx(set_random_backupstore, client, core_api,
volume_name, pvc, csi_pv, pod_make,
make_deployment_with_pvc): # NOQA
"""
Test restoring a rwo to a rwx volume.
1. Create a volume with 'accessMode' rwo.
2. Create a PV and a PVC with access mode 'readwriteonce' and attach to the
volume.
3. Create a pod and attach to the PVC.
4. Write some data into the pod and compute md5sum.
5. Take a backup of the volume.
6. Restore the backup with 'accessMode' rwx.
7. Create PV and PVC and attach to 2 pods.
8. Verify the data.
"""
data_path = "/data/test"
pod_name, pv_name, pvc_name, md5sum = \
prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc,
pod_make,
volume_name,
data_size_in_mb=DATA_SIZE_IN_MB_1,
data_path=data_path)
snap = create_snapshot(client, volume_name)
volume = client.by_id_volume(volume_name)
volume.snapshotBackup(name=snap.name)
wait_for_backup_completion(client, volume_name, snap.name)
bv, b1 = find_backup(client, volume_name, snap.name)
restore_volume_name = 'restored-rwx-volume'
restore_pv_name = restore_volume_name + "-pv"
restore_pvc_name = restore_volume_name + "-pvc"
client.create_volume(name=restore_volume_name,
size=str(1 * Gi),
numberOfReplicas=3,
fromBackup=b1.url,
accessMode='rwx')
wait_for_volume_creation(client, restore_volume_name)
restore_volume = wait_for_volume_detached(client, restore_volume_name)
create_pv_for_volume(client, core_api, restore_volume, restore_pv_name)
create_pvc_for_volume(client, core_api, restore_volume, restore_pvc_name)
deployment = make_deployment_with_pvc('deployment-multi-pods-test',
restore_pvc_name,
replicas=2)
apps_api = get_apps_api_client()
create_and_wait_deployment(apps_api, deployment)
deployment_label_selector = \
"name=" + deployment["metadata"]["labels"]["name"]
deployment_pod_list = \
core_api.list_namespaced_pod(namespace="default",
label_selector=deployment_label_selector)
pod_name_1 = deployment_pod_list.items[0].metadata.name
pod_name_2 = deployment_pod_list.items[1].metadata.name
md5sum_pod1 = get_pod_data_md5sum(core_api, pod_name_1, data_path)
md5sum_pod2 = get_pod_data_md5sum(core_api, pod_name_2, data_path)
assert md5sum == md5sum_pod1 == md5sum_pod2
def test_csi_volumesnapshot_restore_existing_backup(
set_random_backupstore, # NOQA
client, # NOQA
core_api, # NOQA
volume_name, # NOQA
csi_pv, # NOQA
pvc, # NOQA
pod_make, # NOQA
volumesnapshotclass, # NOQA
volumesnapshotcontent,
volumesnapshot, # NOQA
volsnapshotclass_delete_policy, # NOQA
backup_is_deleted): # NOQA
"""
Test retention of a backup while deleting the associated `VolumeSnapshot`
via the csi snapshotter
Context:
We want to allow the user to programmatically create/restore/delete
longhorn backups via the csi snapshot mechanism
ref: https://kubernetes.io/docs/concepts/storage/volume-snapshots/
Setup:
1. Make sure your cluster contains the below crds
https://github.com/kubernetes-csi/external-snapshotter
/tree/master/client/config/crd
2. Make sure your cluster contains the snapshot controller
https://github.com/kubernetes-csi/external-snapshotter
/tree/master/deploy/kubernetes/snapshot-controller
Steps:
1. create new snapshotClass with deletionPolicy set to Retain
2. call csi_volumesnapshot_creation_test(snapshotClass=custom)
3. call csi_volumesnapshot_restore_test()
4. call csi_volumesnapshot_deletion_test(deletionPolicy='Retain'):
5. cleanup
"""
csisnapclass = \
volumesnapshotclass(name="snapshotclass",
deletepolicy=volsnapshotclass_delete_policy)
pod_name, pv_name, pvc_name, md5sum = \
prepare_pod_with_data_in_mb(client, core_api,
csi_pv, pvc, pod_make,
volume_name,
data_path="/data/test")
volume = client.by_id_volume(volume_name)
snap = create_snapshot(client, volume_name)
volume.snapshotBackup(name=snap.name)
wait_for_backup_completion(client, volume_name, snap.name)
bv, b = find_backup(client, volume_name, snap.name)
csivolsnap_name = volume_name + "-volumesnapshot"
csivolsnap_namespace = "default"
volsnapcontent = \
volumesnapshotcontent("volsnapcontent",
csisnapclass["metadata"]["name"],
"Delete",
"bs://" + volume_name + "/" + b.name,
csivolsnap_name,
csivolsnap_namespace)
csivolsnap = volumesnapshot(csivolsnap_name, csivolsnap_namespace,
csisnapclass["metadata"]["name"],
"volumeSnapshotContentName",
volsnapcontent["metadata"]["name"])
restore_pvc_name = pvc["metadata"]["name"] + "-restore"
restore_pvc_size = pvc["spec"]["resources"]["requests"]["storage"]
restore_csi_volume_snapshot(core_api, client, csivolsnap, restore_pvc_name,
restore_pvc_size)
restore_pod = pod_make()
restore_pod_name = restore_pod["metadata"]["name"]
restore_pod['spec']['volumes'] = [create_pvc_spec(restore_pvc_name)]
create_and_wait_pod(core_api, restore_pod)
restore_md5sum = \
get_pod_data_md5sum(core_api, restore_pod_name, path="/data/test")
assert restore_md5sum == md5sum
# Delete volumeSnapshot test
delete_volumesnapshot(csivolsnap["metadata"]["name"], "default")
if backup_is_deleted is False:
find_backup(client, volume_name, b["snapshotName"])
else:
wait_for_backup_delete(client, volume_name, b["name"])
def test_recurring_jobs_for_detached_volume(set_random_backupstore, client,
core_api, apps_api, volume_name,
make_deployment_with_pvc): # NOQA
"""
Test recurring jobs for detached volume
Context:
In the current Longhorn implementation, users cannot do recurring
backup when volumes are detached.
This feature gives the users an option to do recurring backup even when
volumes are detached.
longhorn/longhorn#1509
Steps:
1. Change the setting allow-recurring-job-while-volume-detached to true.
2. Create and attach volume, write 50MB data to the volume.
3. Detach the volume.
4. Set the recurring backup for the volume on every minute.
5. In a 2-minutes retry loop, verify that there is exactly 1 new backup.
6. Delete the recurring backup.
7. Create a PV and PVC from the volume.
8. Create a deployment of 1 pod using the PVC.
9. Write 400MB data to the volume from the pod.
10. Scale down the deployment. Wait until the volume is detached.
11. Set the recurring backup for every 2 minutes.
12. Wait util the recurring backup starts, scale up the deployment to 1
pod.
13. Verify that during the recurring backup, the volume's frontend is
disabled, and pod cannot start.
14. Wait for the recurring backup finishes.
Delete the recurring backup.
15. In a 10-minutes retry loop, verify that the pod can eventually start.
16. Change the setting allow-recurring-job-while-volume-detached to false.
17. Cleanup.
"""
recurring_job_setting = \
client.by_id_setting(SETTING_RECURRING_JOB_WHILE_VOLUME_DETACHED)
client.update(recurring_job_setting, value="true")
vol = common.create_and_check_volume(client, volume_name, size=str(1 * Gi))
lht_hostId = get_self_host_id()
vol.attach(hostId=lht_hostId)
vol = wait_for_volume_healthy(client, vol.name)
data = {
'pos': 0,
'content': common.generate_random_data(50 * Mi),
}
common.write_volume_data(vol, data)
# Give sometimes for data to flush to disk
time.sleep(15)
vol.detach(hostId="")
vol = common.wait_for_volume_detached(client, vol.name)
jobs = [{
"name": RECURRING_JOB_NAME,
"cron": "*/1 * * * *",
"task": "backup",
"retain": 1
}]
vol.recurringUpdate(jobs=jobs)
common.wait_for_backup_completion(client, vol.name)
for _ in range(4):
bv = client.by_id_backupVolume(vol.name)
backups = bv.backupList().data
assert len(backups) == 1
time.sleep(30)
vol.recurringUpdate(jobs=[])
pv_name = volume_name + "-pv"
common.create_pv_for_volume(client, core_api, vol, pv_name)
pvc_name = volume_name + "-pvc"
common.create_pvc_for_volume(client, core_api, vol, pvc_name)
deployment_name = volume_name + "-dep"
deployment = make_deployment_with_pvc(deployment_name, pvc_name)
common.create_and_wait_deployment(apps_api, deployment)
size_mb = 400
pod_names = common.get_deployment_pod_names(core_api, deployment)
write_pod_volume_random_data(core_api, pod_names[0], "/data/test", size_mb)
deployment['spec']['replicas'] = 0
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment["metadata"]["name"])
vol = common.wait_for_volume_detached(client, vol.name)
jobs = [{
"name": RECURRING_JOB_NAME,
"cron": "*/2 * * * *",
"task": "backup",
"retain": 1
}]
vol.recurringUpdate(jobs=jobs)
common.wait_for_backup_to_start(client, vol.name)
deployment['spec']['replicas'] = 1
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment["metadata"]["name"])
deployment_label_name = deployment["metadata"]["labels"]["name"]
common.wait_pod_auto_attach_after_first_backup_completion(
client, core_api, vol.name, deployment_label_name)
vol.recurringUpdate(jobs=[])
pod_names = common.get_deployment_pod_names(core_api, deployment)
common.wait_for_pod_phase(core_api, pod_names[0], pod_phase="Running")