def test_provisioner_io(client, core_api, storage_class, pvc, pod): # NOQA
"""
Test that input and output on a StorageClass provisioned
PersistentVolumeClaim works as expected.
Fixtures are torn down here in reverse order that they are specified as a
parameter. Take caution when reordering test fixtures.
"""
# Prepare pod and volume specs.
pod_name = 'provisioner-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])]
pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME
storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage(core_api, storage_class, pvc)
create_and_wait_pod(core_api, pod)
pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name'])
write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, pvc_volume_name)
pod_name = 'flexvolume-provisioner-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def test_statefulset_pod_deletion(core_api, storage_class, statefulset): # NOQA
"""
Test that a StatefulSet can spin up a new Pod with the same data after a
previous Pod has been deleted.
This test will only work in a CSI environment. It will automatically be
disabled in FlexVolume environments.
"""
statefulset_name = 'statefulset-pod-deletion-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
test_pod_name = statefulset_name + '-' + \
str(randrange(statefulset['spec']['replicas']))
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
write_volume_data(core_api, test_pod_name, test_data)
# Not using delete_and_wait_pod here because there is the small chance the
# StatefulSet recreates the Pod quickly enough where the function won't
# detect that the Pod was deleted, which will time out and throw an error.
core_api.delete_namespaced_pod(name=test_pod_name, namespace='default',
body=k8sclient.V1DeleteOptions())
wait_statefulset(statefulset)
resp = read_volume_data(core_api, test_pod_name)
assert resp == test_data
def test_statefulset_pod_deletion(core_api, storage_class,
statefulset): # NOQA
"""
Test that a StatefulSet can spin up a new Pod with the same data after a
previous Pod has been deleted.
This test will only work in a CSI environment. It will automatically be
disabled in FlexVolume environments.
"""
statefulset_name = 'statefulset-pod-deletion-test'
update_test_manifests(statefulset, storage_class, statefulset_name)
test_pod_name = statefulset_name + '-' + \
str(randrange(statefulset['spec']['replicas']))
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
write_volume_data(core_api, test_pod_name, test_data)
# Not using delete_and_wait_pod here because there is the small chance the
# StatefulSet recreates the Pod quickly enough where the function won't
# detect that the Pod was deleted, which will time out and throw an error.
core_api.delete_namespaced_pod(name=test_pod_name,
namespace='default',
body=k8sclient.V1DeleteOptions())
wait_statefulset(statefulset)
resp = read_volume_data(core_api, test_pod_name)
assert resp == test_data
def test_csi_io(client, core_api, csi_pv, pvc, pod): # NOQA
"""
Test that input and output on a statically defined CSI volume works as
expected.
Fixtures are torn down here in reverse order that they are specified as a
parameter. Take caution when reordering test fixtures.
"""
pod_name = 'csi-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])]
pvc['spec']['volumeName'] = csi_pv['metadata']['name']
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_pv_storage(core_api, client, csi_pv, pvc)
create_and_wait_pod(core_api, pod)
write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, csi_pv['metadata']['name'])
pod_name = 'csi-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def csi_io_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA
pv_name = generate_volume_name()
pod_name = 'csi-io-test'
create_and_wait_csi_pod_named_pv(pv_name, pod_name, client, core_api,
csi_pv, pvc, pod_make, base_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, csi_pv['metadata']['name'])
pod_name = 'csi-io-test-2'
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [
create_pvc_spec(pv_name)
]
csi_pv['metadata']['name'] = pv_name
csi_pv['spec']['csi']['volumeHandle'] = pv_name
pvc['metadata']['name'] = pv_name
pvc['spec']['volumeName'] = pv_name
update_storageclass_references(CSI_PV_TEST_STORAGE_NAME, csi_pv, pvc)
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA
vol_name = csi_pv['metadata']['name']
write_volume_data(core_api, pod_name, test_data)
volume = client.by_id_volume(vol_name)
snap = volume.snapshotCreate()
volume.snapshotBackup(name=snap["name"])
bv, b = common.find_backup(client, vol_name, snap["name"])
pod2_name = 'csi-backup-test-' + str(i)
create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make,
base_image, b["url"])
resp = read_volume_data(core_api, pod2_name)
assert resp == test_data
bv.backupDelete(name=b["name"])
backups = bv.backupList()
found = False
for b in backups:
if b["snapshotName"] == snap["name"]:
found = True
break
assert not found
def test_statefulset_pod_deletion(core_api, storage_class,
statefulset): # NOQA
"""
Test that a StatefulSet can spin up a new Pod with the same data after a
previous Pod has been deleted.
1. Create a StatefulSet with VolumeClaimTemplate and Longhorn.
2. Wait for pods to run.
3. Write some data to one of the pod.
4. Delete that pod.
5. Wait for the StatefulSet to recreate the pod
6. Verify the data in the pod.
"""
statefulset_name = 'statefulset-pod-deletion-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
test_pod_name = statefulset_name + '-' + \
str(randrange(statefulset['spec']['replicas']))
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
write_pod_volume_data(core_api, test_pod_name, test_data)
# Not using delete_and_wait_pod here because there is the small chance the
# StatefulSet recreates the Pod quickly enough where the function won't
# detect that the Pod was deleted, which will time out and throw an error.
core_api.delete_namespaced_pod(name=test_pod_name,
namespace='default',
body=k8sclient.V1DeleteOptions())
wait_statefulset(statefulset)
resp = read_volume_data(core_api, test_pod_name)
assert resp == test_data
def test_flexvolume_io(client, core_api, flexvolume, pod): # NOQA
"""
Test that input and output on a statically defined volume works as
expected.
Fixtures are torn down here in reverse order that they are specified as a
parameter. Take caution when reordering test fixtures.
"""
pod_name = 'flexvolume-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \
flexvolume['name']
pod['spec']['volumes'] = [flexvolume]
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_and_wait_pod(core_api, pod)
common.write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
wait_for_volume_detached(client, flexvolume["name"])
pod_name = 'volume-driver-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name,
base_image, test_data, i): # NOQA
vol_name = csi_pv['metadata']['name']
write_pod_volume_data(core_api, pod_name, test_data)
volume = client.by_id_volume(vol_name)
snap = volume.snapshotCreate()
volume.snapshotBackup(name=snap["name"])
bv, b = common.find_backup(client, vol_name, snap["name"])
pod2_name = 'csi-backup-test-' + str(i)
create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make,
base_image, b["url"])
resp = read_volume_data(core_api, pod2_name)
assert resp == test_data
bv.backupDelete(name=b["name"])
backups = bv.backupList()
found = False
for b in backups:
if b["snapshotName"] == snap["name"]:
found = True
break
assert not found
def test_provisioner_io(client, core_api, storage_class, pvc, pod): # NOQA
"""
Test that input and output on a StorageClass provisioned
PersistentVolumeClaim works as expected.
Fixtures are torn down here in reverse order that they are specified as a
parameter. Take caution when reordering test fixtures.
"""
# Prepare pod and volume specs.
pod_name = 'provisioner-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [
create_pvc_spec(pvc['metadata']['name'])
]
pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME
storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage(core_api, storage_class, pvc)
create_and_wait_pod(core_api, pod)
pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name'])
write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, pvc_volume_name)
pod_name = 'flexvolume-provisioner-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def csi_io_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA
pv_name = generate_volume_name()
pod_name = 'csi-io-test'
create_and_wait_csi_pod_named_pv(pv_name, pod_name, client, core_api,
csi_pv, pvc, pod_make, base_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, csi_pv['metadata']['name'])
pod_name = 'csi-io-test-2'
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [
create_pvc_spec(pv_name)
]
csi_pv['metadata']['name'] = pv_name
csi_pv['spec']['csi']['volumeHandle'] = pv_name
pvc['metadata']['name'] = pv_name
pvc['spec']['volumeName'] = pv_name
update_storageclass_references(CSI_PV_TEST_STORAGE_NAME, csi_pv, pvc)
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def delete_data(k8s_api_client, pod_name):
file_name = 'data-' + pod_name + '.bin'
test_data = generate_random_data(0)
write_pod_volume_data(k8s_api_client,
pod_name,
test_data,
filename=file_name)
volume_data = read_volume_data(k8s_api_client,
pod_name,
filename=file_name)
assert volume_data == ""
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, base_image, test_data, i): # NOQA
vol_name = csi_pv['metadata']['name']
write_pod_volume_data(core_api, pod_name, test_data)
volume = client.by_id_volume(vol_name)
snap = create_snapshot(client, vol_name)
volume.snapshotBackup(name=snap.name)
common.wait_for_backup_completion(client, vol_name, snap.name)
bv, b = common.find_backup(client, vol_name, snap.name)
pod2_name = 'csi-backup-test-' + str(i)
create_and_wait_csi_pod(pod2_name, client, core_api, csi_pv, pvc, pod_make,
base_image, b.url)
resp = read_volume_data(core_api, pod2_name)
assert resp == test_data
delete_backup(client, bv.name, b.name)
def flexvolume_io_test(client, core_api, flexvolume, pod): # NOQA
pod_name = 'flexvolume-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \
flexvolume['name']
pod['spec']['volumes'] = [flexvolume]
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_and_wait_pod(core_api, pod)
common.write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
wait_for_volume_detached(client, flexvolume["name"])
pod_name = 'volume-driver-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, vol_name, backing_image, test_data): # NOQA
write_pod_volume_data(core_api, pod_name, test_data)
volume = client.by_id_volume(vol_name)
snap = create_snapshot(client, vol_name)
volume.snapshotBackup(name=snap.name)
common.wait_for_backup_completion(client, vol_name, snap.name)
bv, b = common.find_backup(client, vol_name, snap.name)
pod2_name = 'csi-backup-test-2'
vol2_name = create_and_wait_csi_pod(
pod2_name, client, core_api, csi_pv, pvc, pod_make,
backing_image, b.url)
volume2 = client.by_id_volume(vol2_name)
resp = read_volume_data(core_api, pod2_name)
assert resp == test_data
delete_backup(client, bv.name, b.name)
delete_and_wait_pod(core_api, pod2_name)
client.delete(volume2)
def flexvolume_io_test(client, core_api, flexvolume, pod): # NOQA
pod_name = 'flexvolume-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \
flexvolume['name']
pod['spec']['volumes'] = [
flexvolume
]
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_and_wait_pod(core_api, pod)
common.write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
wait_for_volume_detached(client, flexvolume["name"])
pod_name = 'volume-driver-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def test_xfs_pv(client, core_api, pod_manifest): # NOQA
"""
Test create PV with new XFS filesystem
1. Create a volume
2. Create a PV for the existing volume, specify `xfs` as filesystem
3. Create PVC and Pod
4. Make sure Pod is running.
5. Write data into the pod and read back for validation.
Note: The volume will be formatted to XFS filesystem by Kubernetes in this
case.
"""
volume_name = generate_volume_name()
volume = create_and_check_volume(client, volume_name)
create_pv_for_volume(client, core_api, volume, volume_name, "xfs")
create_pvc_for_volume(client, core_api, volume, volume_name)
pod_manifest['spec']['volumes'] = [{
"name": "pod-data",
"persistentVolumeClaim": {
"claimName": volume_name
}
}]
pod_name = pod_manifest['metadata']['name']
create_and_wait_pod(core_api, pod_manifest)
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def test_statefulset_restore(client, core_api, storage_class, # NOQA
statefulset): # NOQA
"""
Test that data can be restored into volumes usable by a StatefulSet.
"""
statefulset_name = 'statefulset-restore-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
create_and_test_backups(core_api, client, pod_info)
delete_and_wait_statefulset(core_api, client, statefulset)
csi = check_csi(core_api)
# StatefulSet fixture already cleans these up, use the manifests instead of
# the fixtures to avoid issues during teardown.
pv = {
'apiVersion': 'v1',
'kind': 'PersistentVolume',
'metadata': {
'name': ''
},
'spec': {
'capacity': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
},
'volumeMode': 'Filesystem',
'accessModes': ['ReadWriteOnce'],
'persistentVolumeReclaimPolicy': 'Delete',
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
pvc = {
'apiVersion': 'v1',
'kind': 'PersistentVolumeClaim',
'metadata': {
'name': ''
},
'spec': {
'accessModes': [
'ReadWriteOnce'
],
'resources': {
'requests': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
},
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
if csi:
pv['spec']['csi'] = {
'driver': 'io.rancher.longhorn',
'fsType': 'ext4',
'volumeAttributes': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout']
},
'volumeHandle': ''
}
else:
pv['spec']['flexVolume'] = {
'driver': 'rancher.io/longhorn',
'fsType': 'ext4',
'options': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout'],
'fromBackup': '',
'size': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
}
# Make sure that volumes still work even if the Pod and StatefulSet names
# are different.
for pod in pod_info:
pod['pod_name'] = pod['pod_name'].replace('statefulset-restore-test',
'statefulset-restore-test-2')
pod['pvc_name'] = pod['pvc_name'].replace('statefulset-restore-test',
'statefulset-restore-test-2')
pv['metadata']['name'] = pod['pvc_name']
if csi:
client.create_volume(
name=pod['pvc_name'],
size=size_to_string(DEFAULT_VOLUME_SIZE * Gi),
numberOfReplicas=int(
storage_class['parameters']['numberOfReplicas']),
fromBackup=pod['backup_snapshot']['url'])
wait_for_volume_detached(client, pod['pvc_name'])
pv['spec']['csi']['volumeHandle'] = pod['pvc_name']
else:
pv['spec']['flexVolume']['options']['fromBackup'] = \
pod['backup_snapshot']['url']
core_api.create_persistent_volume(pv)
pvc['metadata']['name'] = pod['pvc_name']
pvc['spec']['volumeName'] = pod['pvc_name']
core_api.create_namespaced_persistent_volume_claim(
body=pvc,
namespace='default')
statefulset_name = 'statefulset-restore-test-2'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_and_wait_statefulset(statefulset)
for pod in pod_info:
resp = read_volume_data(core_api, pod['pod_name'])
assert resp == pod['data']
def test_csi_minimal_volume_size(
client, core_api, csi_pv, pvc, pod_make): # NOQA
"""
Test CSI Minimal Volume Size
1. Create a PVC requesting size 5MiB. Check the PVC requested size is
5MiB and capacity size get is 10MiB.
2. Remove the PVC.
3. Create a PVC requesting size 10MiB. Check the PVC requested size and
capacity size get are both 10MiB.
4. Create a pod to use this PVC.
5. Write some data to the volume and read it back to compare.
"""
vol_name = generate_volume_name()
create_and_check_volume(client, vol_name, size=str(100*Mi))
low_storage = str(5*Mi)
min_storage = str(10*Mi)
pv_name = vol_name + "-pv"
csi_pv['metadata']['name'] = pv_name
csi_pv['spec']['csi']['volumeHandle'] = vol_name
csi_pv['spec']['capacity']['storage'] = min_storage
core_api.create_persistent_volume(csi_pv)
pvc_name = vol_name + "-pvc"
pvc['metadata']['name'] = pvc_name
pvc['spec']['volumeName'] = pv_name
pvc['spec']['resources']['requests']['storage'] = low_storage
pvc['spec']['storageClassName'] = ''
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
claim = common.wait_for_pvc_phase(core_api, pvc_name, "Bound")
assert claim.spec.resources.requests['storage'] == low_storage
assert claim.status.capacity['storage'] == min_storage
common.delete_and_wait_pvc(core_api, pvc_name)
common.delete_and_wait_pv(core_api, pv_name)
wait_for_volume_detached(client, vol_name)
core_api.create_persistent_volume(csi_pv)
pvc['spec']['resources']['requests']['storage'] = min_storage
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
claim = common.wait_for_pvc_phase(core_api, pvc_name, "Bound")
assert claim.spec.resources.requests['storage'] == min_storage
assert claim.status.capacity['storage'] == min_storage
pod_name = vol_name + '-pod'
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
create_and_wait_pod(core_api, pod)
test_data = "longhorn-integration-test"
test_file = "test"
write_pod_volume_data(core_api, pod_name, test_data, test_file)
read_data = read_volume_data(core_api, pod_name, test_file)
assert read_data == test_data
def test_statefulset_restore(
client,
core_api,
storage_class, # NOQA
statefulset): # NOQA
"""
Test that data can be restored into volumes usable by a StatefulSet.
1. Create a StatefulSet with VolumeClaimTemplate and Longhorn.
2. Wait for pods to run.
3. Create a backup for each pod.
4. Delete the StatefulSet, including the Longhorn volumes.
5. Create volumes and PV/PVC using previous backups from each Pod.
1. PVs will be created using the previous names.
2. PVCs will be created using previous name + "-2" due to statefulset
has a naming policy for what should be PVC name for them.
6. Create a new StatefulSet using the previous name + "-2"
7. Wait for pods to be up.
. Verify the pods contain the previous backed up data
"""
statefulset_name = 'statefulset-restore-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
create_and_test_backups(core_api, client, pod_info)
delete_and_wait_statefulset(core_api, client, statefulset)
csi = check_csi(core_api)
# StatefulSet fixture already cleans these up, use the manifests instead of
# the fixtures to avoid issues during teardown.
pv = {
'apiVersion': 'v1',
'kind': 'PersistentVolume',
'metadata': {
'name': ''
},
'spec': {
'capacity': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
},
'volumeMode': 'Filesystem',
'accessModes': ['ReadWriteOnce'],
'persistentVolumeReclaimPolicy': 'Delete',
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
pvc = {
'apiVersion': 'v1',
'kind': 'PersistentVolumeClaim',
'metadata': {
'name': ''
},
'spec': {
'accessModes': ['ReadWriteOnce'],
'resources': {
'requests': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
},
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
assert csi
pv['spec']['csi'] = {
'driver': 'driver.longhorn.io',
'fsType': 'ext4',
'volumeAttributes': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout']
},
'volumeHandle': ''
}
# Make sure that volumes still work even if the Pod and StatefulSet names
# are different.
for pod in pod_info:
pod['pod_name'] = pod['pod_name'].replace(
'statefulset-restore-test', 'statefulset-restore-test-2')
pod['pvc_name'] = pod['pvc_name'].replace(
'statefulset-restore-test', 'statefulset-restore-test-2')
pv['metadata']['name'] = pod['pvc_name']
client.create_volume(
name=pod['pvc_name'],
size=size_to_string(DEFAULT_VOLUME_SIZE * Gi),
numberOfReplicas=int(
storage_class['parameters']['numberOfReplicas']),
fromBackup=pod['backup_snapshot']['url'])
wait_for_volume_detached(client, pod['pvc_name'])
pv['spec']['csi']['volumeHandle'] = pod['pvc_name']
core_api.create_persistent_volume(pv)
pvc['metadata']['name'] = pod['pvc_name']
pvc['spec']['volumeName'] = pod['pvc_name']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
statefulset_name = 'statefulset-restore-test-2'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_and_wait_statefulset(statefulset)
for pod in pod_info:
resp = read_volume_data(core_api, pod['pod_name'])
assert resp == pod['data']
def test_upgrade(upgrade_image_tag, settings_reset, volume_name, pod_make, statefulset, storage_class): # NOQA
"""
Test Longhorn upgrade
Prerequisite:
- Disable Auto Salvage Setting
1. Find the upgrade image tag
2. Create a volume, generate and write data into the volume.
3. Create a Pod using a volume, generate and write data
4. Create a StatefulSet with 2 replicas,
generate and write data to their volumes
5. Keep all volumes attached
6. Upgrade Longhorn system.
7. Check Pod and StatefulSet didn't restart after upgrade
8. Check All volumes data
9. Write data to StatefulSet pods, and Attached volume
10. Check data written to StatefulSet pods, and attached volume.
11. Detach the volume, and Delete Pod, and
StatefulSet to detach theirvolumes
12. Upgrade all volumes engine images.
13. Attach the volume, and recreate Pod, and StatefulSet
14. Check All volumes data
"""
new_ei_name = "longhornio/longhorn-engine:" + upgrade_image_tag
client = get_longhorn_api_client()
core_api = get_core_api_client()
host_id = get_self_host_id()
pod_data_path = "/data/test"
pod_volume_name = generate_volume_name()
auto_salvage_setting = client.by_id_setting(SETTING_AUTO_SALVAGE)
setting = client.update(auto_salvage_setting, value="false")
assert setting.name == SETTING_AUTO_SALVAGE
assert setting.value == "false"
# Create Volume attached to a node.
volume1 = create_and_check_volume(client,
volume_name,
size=SIZE)
volume1.attach(hostId=host_id)
volume1 = wait_for_volume_healthy(client, volume_name)
volume1_data = write_volume_random_data(volume1)
# Create Volume used by Pod
pod_name, pv_name, pvc_name, pod_md5sum = \
prepare_pod_with_data_in_mb(client, core_api,
pod_make, pod_volume_name,
data_path=pod_data_path,
add_liveness_prope=False)
# Create multiple volumes used by StatefulSet
statefulset_name = 'statefulset-upgrade-test'
update_statefulset_manifests(statefulset,
storage_class,
statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
statefulset_pod_info = get_statefulset_pod_info(core_api, statefulset)
for sspod_info in statefulset_pod_info:
sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api,
sspod_info['pod_name'],
sspod_info['data'])
# upgrade Longhorn
assert longhorn_upgrade(upgrade_image_tag)
client = get_longhorn_api_client()
# wait for 1 minute before checking pod restarts
time.sleep(60)
pod = core_api.read_namespaced_pod(name=pod_name,
namespace='default')
assert pod.status.container_statuses[0].restart_count == 0
for sspod_info in statefulset_pod_info:
sspod = core_api.read_namespaced_pod(name=sspod_info['pod_name'],
namespace='default')
assert \
sspod.status.container_statuses[0].restart_count == 0
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
assert res_pod_md5sum == pod_md5sum
check_volume_data(volume1, volume1_data)
for sspod_info in statefulset_pod_info:
sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api,
sspod_info['pod_name'],
sspod_info['data'])
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
volume1 = client.by_id_volume(volume_name)
volume1_data = write_volume_random_data(volume1)
check_volume_data(volume1, volume1_data)
statefulset['spec']['replicas'] = replicas = 0
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={
'spec': {
'replicas': replicas
}
})
delete_and_wait_pod(core_api, pod_name)
volume = client.by_id_volume(volume_name)
volume.detach()
volumes = client.list_volume()
for v in volumes:
wait_for_volume_detached(client, v.name)
engineimages = client.list_engine_image()
for ei in engineimages:
if ei.image == new_ei_name:
new_ei = ei
volumes = client.list_volume()
for v in volumes:
volume = client.by_id_volume(v.name)
volume.engineUpgrade(image=new_ei.image)
statefulset['spec']['replicas'] = replicas = 2
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={
'spec': {
'replicas': replicas
}
})
wait_statefulset(statefulset)
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
create_and_wait_pod(core_api, pod)
volume1 = client.by_id_volume(volume_name)
volume1.attach(hostId=host_id)
volume1 = wait_for_volume_healthy(client, volume_name)
for sspod_info in statefulset_pod_info:
resp = read_volume_data(core_api, sspod_info['pod_name'])
assert resp == sspod_info['data']
res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
assert res_pod_md5sum == pod_md5sum
check_volume_data(volume1, volume1_data)
def test_offline_node_with_attached_volume_and_pod(
client, core_api, volume_name, make_deployment_with_pvc,
reset_cluster_ready_status): # NOQA
"""
Test offline node with attached volume and pod
1. Create PV/PVC/Deployment manifest.
2. Update deployment's tolerations to 20 seconds to speed up test
3. Update deployment's node affinity rule to avoid the current node
4. Create volume, PV/PVC and deployment.
5. Find the pod in the deployment and write `test_data` into it
6. Shutdown the node pod is running on
7. Wait for deployment to delete the pod
1. Deployment cannot delete the pod here because kubelet doesn't
response
8. Force delete the terminating pod
9. Wait for the new pod to be created and the volume attached
10. Check `test_data` in the new pod
"""
toleration_seconds = 20
apps_api = get_apps_api_client()
cloudprovider = detect_cloudprovider()
volume_name = generate_volume_name()
pv_name = volume_name + "-pv"
pvc_name = volume_name + "-pvc"
deployment_name = volume_name + "-dep"
longhorn_test_node_name = get_self_host_id()
deployment_manifest = make_deployment_with_pvc(deployment_name, pvc_name)
unreachable_toleration = {
"key": "node.kubernetes.io/unreachable",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": toleration_seconds
}
not_ready_toleration = {
"key": "node.kubernetes.io/not-ready",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": toleration_seconds
}
deployment_manifest["spec"]["template"]["spec"]["tolerations"] =\
[unreachable_toleration, not_ready_toleration]
node_affinity_roles = {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kubernetes.io/hostname",
"operator": "NotIn",
"values": [longhorn_test_node_name]
}]
}]
}
}
}
deployment_manifest["spec"]["template"]["spec"]["affinity"] =\
node_affinity_roles
longhorn_volume = create_and_check_volume(client, volume_name, size=SIZE)
wait_for_volume_detached(client, volume_name)
create_pv_for_volume(client, core_api, longhorn_volume, pv_name)
create_pvc_for_volume(client, core_api, longhorn_volume, pvc_name)
create_and_wait_deployment(apps_api, deployment_manifest)
deployment_label_selector =\
"name=" + deployment_manifest["metadata"]["labels"]["name"]
deployment_pod_list =\
core_api.list_namespaced_pod(namespace="default",
label_selector=deployment_label_selector)
assert deployment_pod_list.items.__len__() == 1
pod_name = deployment_pod_list.items[0].metadata.name
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
node_name = deployment_pod_list.items[0].spec.node_name
node = cloudprovider.node_id(node_name)
cloudprovider.node_shutdown(node)
k8s_node_down = wait_for_node_down_k8s(node_name, core_api)
assert k8s_node_down
client = get_longhorn_api_client()
longhorn_node_down = wait_for_node_down_longhorn(node_name, client)
assert longhorn_node_down
time.sleep(toleration_seconds + 5)
for i in range(TERMINATING_POD_RETRYS):
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
terminating_pod_name = None
for pod in deployment_pod_list.items:
if pod.metadata.__getattribute__("deletion_timestamp") is not None:
terminating_pod_name = pod.metadata.name
break
if terminating_pod_name is not None:
break
else:
time.sleep(TERMINATING_POD_INTERVAL)
assert terminating_pod_name is not None
core_api.delete_namespaced_pod(namespace="default",
name=terminating_pod_name,
grace_period_seconds=0)
delete_and_wait_pod(core_api, terminating_pod_name)
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
assert deployment_pod_list.items.__len__() == 1
wait_for_volume_detached(client, volume_name)
wait_for_volume_healthy(client, volume_name)
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
assert deployment_pod_list.items.__len__() == 1
new_pod_name = deployment_pod_list.items[0].metadata.name
wait_pod(new_pod_name)
resp_data = read_volume_data(core_api, new_pod_name)
assert test_data == resp_data
def test_statefulset_restore(
client,
core_api,
storage_class, # NOQA
statefulset): # NOQA
"""
Test that data can be restored into volumes usable by a StatefulSet.
"""
statefulset_name = 'statefulset-restore-test'
update_test_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_info = get_statefulset_pod_info(core_api, statefulset)
create_and_test_backups(core_api, client, pod_info)
delete_and_wait_statefulset(core_api, client, statefulset)
csi = True
try:
core_api.read_namespaced_pod(name='csi-provisioner-0',
namespace='longhorn-system')
except ApiException as e:
if (e.status == 404):
csi = False
# StatefulSet fixture already cleans these up, use the manifests instead of
# the fixtures to avoid issues during teardown.
pv = {
'apiVersion': 'v1',
'kind': 'PersistentVolume',
'metadata': {
'name': ''
},
'spec': {
'capacity': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
},
'volumeMode': 'Filesystem',
'accessModes': ['ReadWriteOnce'],
'persistentVolumeReclaimPolicy': 'Delete',
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
pvc = {
'apiVersion': 'v1',
'kind': 'PersistentVolumeClaim',
'metadata': {
'name': ''
},
'spec': {
'accessModes': ['ReadWriteOnce'],
'resources': {
'requests': {
'storage': size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
},
'storageClassName': DEFAULT_STORAGECLASS_NAME
}
}
if csi:
pv['spec']['csi'] = {
'driver': 'io.rancher.longhorn',
'fsType': 'ext4',
'volumeAttributes': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout']
},
'volumeHandle': ''
}
else:
pv['spec']['flexVolume'] = {
'driver': 'rancher.io/longhorn',
'fsType': 'ext4',
'options': {
'numberOfReplicas':
storage_class['parameters']['numberOfReplicas'],
'staleReplicaTimeout':
storage_class['parameters']['staleReplicaTimeout'],
'fromBackup':
'',
'size':
size_to_string(DEFAULT_VOLUME_SIZE * Gi)
}
}
# Make sure that volumes still work even if the Pod and StatefulSet names
# are different.
for pod in pod_info:
pod['pod_name'] = pod['pod_name'].replace(
'statefulset-restore-test', 'statefulset-restore-test-2')
pod['pvc_name'] = pod['pvc_name'].replace(
'statefulset-restore-test', 'statefulset-restore-test-2')
pv['metadata']['name'] = pod['pvc_name']
if csi:
client.create_volume(
name=pod['pvc_name'],
size=size_to_string(DEFAULT_VOLUME_SIZE * Gi),
numberOfReplicas=int(
storage_class['parameters']['numberOfReplicas']),
fromBackup=pod['backup_snapshot']['url'])
wait_for_volume_detached(client, pod['pvc_name'])
pv['spec']['csi']['volumeHandle'] = pod['pvc_name']
else:
pv['spec']['flexVolume']['options']['fromBackup'] = \
pod['backup_snapshot']['url']
core_api.create_persistent_volume(pv)
pvc['metadata']['name'] = pod['pvc_name']
pvc['spec']['volumeName'] = pod['pvc_name']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
statefulset_name = 'statefulset-restore-test-2'
update_test_manifests(statefulset, storage_class, statefulset_name)
create_and_wait_statefulset(statefulset)
for pod in pod_info:
resp = read_volume_data(core_api, pod['pod_name'])
assert resp == pod['data']
def test_csi_offline_expansion(client, core_api, storage_class, pvc, pod_manifest): # NOQA
"""
Test CSI feature: offline expansion
1. Create a new `storage_class` with `allowVolumeExpansion` set
2. Create PVC and Pod with dynamic provisioned volume from the StorageClass
3. Generate `test_data` and write to the pod
4. Delete the pod
5. Update pvc.spec.resources to expand the volume
6. Verify the volume expansion done using Longhorn API
7. Create a new pod and validate the volume content
"""
create_storage_class(storage_class)
pod_name = 'csi-offline-expand-volume-test'
pvc_name = pod_name + "-pvc"
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = storage_class['metadata']['name']
create_pvc(pvc)
pod_manifest['metadata']['name'] = pod_name
pod_manifest['spec']['volumes'] = [{
'name':
pod_manifest['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {'claimName': pvc_name},
}]
create_and_wait_pod(core_api, pod_manifest)
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
pv = wait_and_get_pv_for_pvc(core_api, pvc_name)
assert pv.status.phase == "Bound"
volume_name = pv.spec.csi.volume_handle
wait_for_volume_detached(client, volume_name)
pvc['spec']['resources'] = {
'requests': {
'storage': size_to_string(EXPANDED_VOLUME_SIZE*Gi)
}
}
expand_and_wait_for_pvc(core_api, pvc)
wait_for_volume_expansion(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume.state == "detached"
assert volume.size == str(EXPANDED_VOLUME_SIZE*Gi)
pod_manifest['metadata']['name'] = pod_name
pod_manifest['spec']['volumes'] = [{
'name':
pod_manifest['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {'claimName': pvc_name},
}]
create_and_wait_pod(core_api, pod_manifest)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
volume = client.by_id_volume(volume_name)
engine = get_volume_engine(volume)
assert volume.size == str(EXPANDED_VOLUME_SIZE*Gi)
assert volume.size == engine.size
def test_rwx_multi_statefulset_with_same_pvc(core_api, pvc, statefulset,
pod): # NOQA
"""
Test writing of data into a volume from multiple pods using same PVC
1. Create a volume with 'accessMode' rwx.
2. Create a PV and a PVC with access mode 'readwritemany' and attach to the
volume.
3. Deploy a StatefulSet of 2 pods with the existing PVC above created.
4. Wait for both pods to come up.
5. Create a pod with the existing PVC above created.
6. Wait for StatefulSet to come up healthy.
7. Write data all three pods and compute md5sum.
8. Check the data md5sum in the share manager pod.
"""
pvc_name = 'pvc-multi-pods-test'
statefulset_name = 'statefulset-rwx-same-pvc-test'
pod_name = 'pod-rwx-same-pvc-test'
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = 'longhorn'
pvc['spec']['accessModes'] = ['ReadWriteMany']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
statefulset['metadata']['name'] = \
statefulset['spec']['selector']['matchLabels']['app'] = \
statefulset['spec']['serviceName'] = \
statefulset['spec']['template']['metadata']['labels']['app'] = \
statefulset_name
statefulset['spec']['template']['spec']['volumes'] = \
[create_pvc_spec(pvc_name)]
del statefulset['spec']['volumeClaimTemplates']
create_and_wait_statefulset(statefulset)
pv_name = get_volume_name(core_api, pvc_name)
share_manager_name = 'share-manager-' + pv_name
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api,
statefulset_name + '-0',
test_data,
filename='test1')
assert test_data == read_volume_data(core_api,
statefulset_name + '-1',
filename='test1')
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
create_and_wait_pod(core_api, pod)
assert test_data == read_volume_data(core_api, pod_name, filename='test1')
test_data_2 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data_2, filename='test2')
command1 = 'cat /export' + '/' + pv_name + '/' + 'test1'
command2 = 'cat /export' + '/' + pv_name + '/' + 'test2'
assert test_data == exec_command_in_pod(core_api, command1,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_2 == exec_command_in_pod(core_api, command2,
share_manager_name,
LONGHORN_NAMESPACE)
def test_csi_expansion_with_replica_failure(client, core_api, storage_class, pvc, pod_manifest): # NOQA
"""
Test expansion success but with one replica expansion failure
1. Create a new `storage_class` with `allowVolumeExpansion` set
2. Create PVC and Pod with dynamic provisioned volume from the StorageClass
3. Create an empty directory with expansion snapshot tmp meta file path
for one replica so that the replica expansion will fail
4. Generate `test_data` and write to the pod
5. Delete the pod and wait for volume detachment
6. Update pvc.spec.resources to expand the volume
7. Check expansion result using Longhorn API. There will be expansion error
caused by the failed replica but overall the expansion should succeed.
8. Create a new pod and
check if the volume will rebuild the failed replica
9. Validate the volume content, then check if data writing looks fine
"""
create_storage_class(storage_class)
pod_name = 'csi-expansion-with-replica-failure-test'
pvc_name = pod_name + "-pvc"
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = storage_class['metadata']['name']
create_pvc(pvc)
pod_manifest['metadata']['name'] = pod_name
pod_manifest['spec']['volumes'] = [{
'name':
pod_manifest['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {'claimName': pvc_name},
}]
create_and_wait_pod(core_api, pod_manifest)
expand_size = str(EXPANDED_VOLUME_SIZE*Gi)
pv = wait_and_get_pv_for_pvc(core_api, pvc_name)
assert pv.status.phase == "Bound"
volume_name = pv.spec.csi.volume_handle
volume = client.by_id_volume(volume_name)
failed_replica = volume.replicas[0]
fail_replica_expansion(client, core_api,
volume_name, expand_size, [failed_replica])
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
wait_for_volume_detached(client, volume_name)
# There will be replica expansion error info
# but the expansion should succeed.
pvc['spec']['resources'] = {
'requests': {
'storage': size_to_string(EXPANDED_VOLUME_SIZE*Gi)
}
}
expand_and_wait_for_pvc(core_api, pvc)
wait_for_expansion_failure(client, volume_name)
wait_for_volume_expansion(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume.state == "detached"
assert volume.size == expand_size
for r in volume.replicas:
if r.name == failed_replica.name:
assert r.failedAt != ""
else:
assert r.failedAt == ""
# Check if the replica will be rebuilded
# and if the volume still works fine.
create_and_wait_pod(core_api, pod_manifest)
volume = wait_for_volume_healthy(client, volume_name)
for r in volume.replicas:
if r.name == failed_replica.name:
assert r.mode == ""
else:
assert r.mode == "RW"
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def test_recurring_jobs_when_volume_detached_unexpectedly(
settings_reset, set_random_backupstore, client, core_api, apps_api,
pvc, make_deployment_with_pvc): # NOQA
"""
Test recurring jobs when volume detached unexpectedly
Context:
If the volume is automatically attached by the recurring backup job,
make sure that workload pod eventually is able to use the volume
when volume is detached unexpectedly during the backup process.
Steps:
1. Create a volume, attach to a pod of a deployment,
write 500MB to the volume.
2. Scale down the deployment. The volume is detached.
3. Turn on `Allow Recurring Job While Volume Is Detached` setting.
4. Create a recurring backup job that runs every 2 mins.
5. Wait until the recurring backup job starts and the backup progress
is > 50%, kill the engine process of the volume.
6. Verify volume automatically reattached and is healthy again.
7. Wait until the backup finishes.
8. Wait for the volume to be in detached state with
`frontendDisabled=false`
9. Scale up the deployment.
Verify that we can read the file `lost+found` from the workload pod
10. Turn off `Allow Recurring Job While Volume Is Detached` setting
Clean up backups, volumes.
"""
recurring_job_setting = \
client.by_id_setting(SETTING_RECURRING_JOB_WHILE_VOLUME_DETACHED)
client.update(recurring_job_setting, value="true")
pvc_name = 'pvc-volume-detached-unexpectedly-test'
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = 'longhorn'
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
deployment = make_deployment_with_pvc(
'deployment-volume-detached-unexpectedly-test', pvc_name)
create_and_wait_deployment(apps_api, deployment)
pod_names = common.get_deployment_pod_names(core_api, deployment)
vol_name = get_volume_name(core_api, pvc_name)
write_pod_volume_random_data(core_api, pod_names[0], "/data/test",
DATA_SIZE_IN_MB_3)
data = read_volume_data(core_api, pod_names[0], 'default')
deployment['spec']['replicas'] = 0
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment["metadata"]["name"])
vol = wait_for_volume_detached(client, vol_name)
jobs = [{
"name": RECURRING_JOB_NAME,
"cron": "*/2 * * * *",
"task": "backup",
"retain": 1
}]
vol.recurringUpdate(jobs=jobs)
time.sleep(60)
wait_for_recurring_backup_to_start(client,
core_api,
vol_name,
expected_snapshot_count=1,
minimum_progress=50)
crash_engine_process_with_sigkill(client, core_api, vol_name)
# Check if the volume is reattached after recurring backup is interrupted
time.sleep(10)
wait_for_volume_healthy_no_frontend(client, vol_name)
# Since the backup state is removed after the backup complete and it
# could happen quickly. Checking for the both in-progress and complete
# state could be hard to catch, thus we only check the complete state
def backup_complete_predicate(b):
return b.state == "complete" and b.error == ""
common.wait_for_backup_state(client, vol_name, backup_complete_predicate)
wait_for_volume_detached(client, vol_name)
deployment['spec']['replicas'] = 1
apps_api.patch_namespaced_deployment(body=deployment,
namespace='default',
name=deployment["metadata"]["name"])
wait_deployment_replica_ready(apps_api, deployment["metadata"]["name"], 1)
pod_names = common.get_deployment_pod_names(core_api, deployment)
assert read_volume_data(core_api, pod_names[0], 'default') == data
# Use fixture to cleanup the backupstore and since we
# crashed the engine replica initiated the backup, it's
# backupstore lock will still be present, so we need
# to wait till the lock is expired, before we can delete
# the backups
vol.recurringUpdate(jobs=[])
backupstore.backupstore_wait_for_lock_expiration()
