def test_snapshot_rm_basic(dev): # NOQA
existings = {}
snap1 = Snapshot(dev, generate_random_data(existings))
snap2 = Snapshot(dev, generate_random_data(existings))
snap3 = Snapshot(dev, generate_random_data(existings))
info = cmd.snapshot_info()
assert len(info) == 4
assert VOLUME_HEAD in info
assert snap1.name in info
assert snap2.name in info
assert snap3.name in info
cmd.snapshot_rm(snap2.name)
cmd.snapshot_purge()
info = cmd.snapshot_info()
assert len(info) == 3
assert snap1.name in info
assert snap3.name in info
snap3.verify_checksum()
snap2.verify_data()
snap1.verify_data()
cmd.snapshot_revert(snap1.name)
snap3.refute_data()
snap2.refute_data()
snap1.verify_checksum()
def test_provisioner_io(client, core_api, storage_class, pvc, pod): # NOQA
"""
Test that input and output on a StorageClass provisioned
PersistentVolumeClaim works as expected.
Fixtures are torn down here in reverse order that they are specified as a
parameter. Take caution when reordering test fixtures.
"""
# Prepare pod and volume specs.
pod_name = 'provisioner-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])]
pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME
storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage(core_api, storage_class, pvc)
create_and_wait_pod(core_api, pod)
pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name'])
write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, pvc_volume_name)
pod_name = 'flexvolume-provisioner-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def test_statefulset_pod_deletion(core_api, storage_class,
statefulset): # NOQA
"""
Test that a StatefulSet can spin up a new Pod with the same data after a
previous Pod has been deleted.
This test will only work in a CSI environment. It will automatically be
disabled in FlexVolume environments.
"""
statefulset_name = 'statefulset-pod-deletion-test'
update_test_manifests(statefulset, storage_class, statefulset_name)
test_pod_name = statefulset_name + '-' + \
str(randrange(statefulset['spec']['replicas']))
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
write_volume_data(core_api, test_pod_name, test_data)
# Not using delete_and_wait_pod here because there is the small chance the
# StatefulSet recreates the Pod quickly enough where the function won't
# detect that the Pod was deleted, which will time out and throw an error.
core_api.delete_namespaced_pod(name=test_pod_name,
namespace='default',
body=k8sclient.V1DeleteOptions())
wait_statefulset(statefulset)
resp = read_volume_data(core_api, test_pod_name)
assert resp == test_data
def csi_io_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA
pv_name = generate_volume_name()
pod_name = 'csi-io-test'
create_and_wait_csi_pod_named_pv(pv_name, pod_name, client, core_api,
csi_pv, pvc, pod_make, base_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, csi_pv['metadata']['name'])
pod_name = 'csi-io-test-2'
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [
create_pvc_spec(pv_name)
]
csi_pv['metadata']['name'] = pv_name
csi_pv['spec']['csi']['volumeHandle'] = pv_name
pvc['metadata']['name'] = pv_name
pvc['spec']['volumeName'] = pv_name
update_storageclass_references(CSI_PV_TEST_STORAGE_NAME, csi_pv, pvc)
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def csi_backup_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA
pod_name = 'csi-backup-test'
create_and_wait_csi_pod(pod_name, client, core_api, csi_pv, pvc, pod_make,
base_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
i = 1
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name,
base_image, test_data, i)
i += 1
def test_flexvolume_io(client, core_api, flexvolume, pod): # NOQA
"""
Test that input and output on a statically defined volume works as
expected.
Fixtures are torn down here in reverse order that they are specified as a
parameter. Take caution when reordering test fixtures.
"""
pod_name = 'flexvolume-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \
flexvolume['name']
pod['spec']['volumes'] = [flexvolume]
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_and_wait_pod(core_api, pod)
common.write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
wait_for_volume_detached(client, flexvolume["name"])
pod_name = 'volume-driver-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def test_statefulset_recurring_backup(set_random_backupstore, client, core_api, storage_class, statefulset): # NOQA
"""
Scenario : test recurring backups on StatefulSets
Given 1 default backup recurring jobs created.
When create a statefulset.
And write data to every statefulset pod.
And wait for 5 minutes.
Then 2 snapshots created for every statefulset pod.
"""
# backup every minute
recurring_jobs = {
"backup": {
"task": "backup",
"groups": ["default"],
"cron": "* * * * *",
"retain": 2,
"concurrency": 2,
"labels": {},
},
}
create_recurring_jobs(client, recurring_jobs)
check_recurring_jobs(client, recurring_jobs)
statefulset_name = 'statefulset-backup-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
pod_data = get_statefulset_pod_info(core_api, statefulset)
for pod in pod_data:
pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
pod['backup_snapshot'] = ''
for pod in pod_data:
volume = client.by_id_volume(pod['pv_name'])
write_pod_volume_data(core_api, pod['pod_name'], pod['data'])
time.sleep(150)
for pod in pod_data:
volume = client.by_id_volume(pod['pv_name'])
write_pod_volume_data(core_api, pod['pod_name'], pod['data'])
time.sleep(150)
for pod in pod_data:
volume = client.by_id_volume(pod['pv_name'])
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot.removed is False:
count += 1
# one backup + volume-head
assert count == 2
def test_provisioner_io(client, core_api, storage_class, pvc, pod): # NOQA
"""
Test that input and output on a StorageClass provisioned
PersistentVolumeClaim works as expected.
Fixtures are torn down here in reverse order that they are specified as a
parameter. Take caution when reordering test fixtures.
"""
# Prepare pod and volume specs.
pod_name = 'provisioner-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [
create_pvc_spec(pvc['metadata']['name'])
]
pvc['spec']['storageClassName'] = DEFAULT_STORAGECLASS_NAME
storage_class['metadata']['name'] = DEFAULT_STORAGECLASS_NAME
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage(core_api, storage_class, pvc)
create_and_wait_pod(core_api, pod)
pvc_volume_name = get_volume_name(core_api, pvc['metadata']['name'])
write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, pvc_volume_name)
pod_name = 'flexvolume-provisioner-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def csi_io_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA
pv_name = generate_volume_name()
pod_name = 'csi-io-test'
create_and_wait_csi_pod_named_pv(pv_name, pod_name, client, core_api,
csi_pv, pvc, pod_make, base_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, csi_pv['metadata']['name'])
pod_name = 'csi-io-test-2'
pod = pod_make(name=pod_name)
pod['spec']['volumes'] = [
create_pvc_spec(pv_name)
]
csi_pv['metadata']['name'] = pv_name
csi_pv['spec']['csi']['volumeHandle'] = pv_name
pvc['metadata']['name'] = pv_name
pvc['spec']['volumeName'] = pv_name
update_storageclass_references(CSI_PV_TEST_STORAGE_NAME, csi_pv, pvc)
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def test_statefulset_pod_deletion(core_api, storage_class,
statefulset): # NOQA
"""
Test that a StatefulSet can spin up a new Pod with the same data after a
previous Pod has been deleted.
1. Create a StatefulSet with VolumeClaimTemplate and Longhorn.
2. Wait for pods to run.
3. Write some data to one of the pod.
4. Delete that pod.
5. Wait for the StatefulSet to recreate the pod
6. Verify the data in the pod.
"""
statefulset_name = 'statefulset-pod-deletion-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
test_pod_name = statefulset_name + '-' + \
str(randrange(statefulset['spec']['replicas']))
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
write_pod_volume_data(core_api, test_pod_name, test_data)
# Not using delete_and_wait_pod here because there is the small chance the
# StatefulSet recreates the Pod quickly enough where the function won't
# detect that the Pod was deleted, which will time out and throw an error.
core_api.delete_namespaced_pod(name=test_pod_name,
namespace='default',
body=k8sclient.V1DeleteOptions())
wait_statefulset(statefulset)
resp = read_volume_data(core_api, test_pod_name)
assert resp == test_data
def csi_backup_test(client, core_api, csi_pv, pvc, pod_make, base_image=""): # NOQA
pod_name = 'csi-backup-test'
create_and_wait_csi_pod(pod_name, client, core_api, csi_pv, pvc, pod_make,
base_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
i = 1
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting.value == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential.value == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting.value == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential.value == ""
backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name,
base_image, test_data, i)
i += 1
def test_statefulset_pod_deletion(core_api, storage_class, statefulset): # NOQA
"""
Test that a StatefulSet can spin up a new Pod with the same data after a
previous Pod has been deleted.
This test will only work in a CSI environment. It will automatically be
disabled in FlexVolume environments.
"""
statefulset_name = 'statefulset-pod-deletion-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
test_pod_name = statefulset_name + '-' + \
str(randrange(statefulset['spec']['replicas']))
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
write_volume_data(core_api, test_pod_name, test_data)
# Not using delete_and_wait_pod here because there is the small chance the
# StatefulSet recreates the Pod quickly enough where the function won't
# detect that the Pod was deleted, which will time out and throw an error.
core_api.delete_namespaced_pod(name=test_pod_name, namespace='default',
body=k8sclient.V1DeleteOptions())
wait_statefulset(statefulset)
resp = read_volume_data(core_api, test_pod_name)
assert resp == test_data
def test_csi_io(client, core_api, csi_pv, pvc, pod): # NOQA
"""
Test that input and output on a statically defined CSI volume works as
expected.
Fixtures are torn down here in reverse order that they are specified as a
parameter. Take caution when reordering test fixtures.
"""
pod_name = 'csi-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(pvc['metadata']['name'])]
pvc['spec']['volumeName'] = csi_pv['metadata']['name']
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_pv_storage(core_api, client, csi_pv, pvc)
create_and_wait_pod(core_api, pod)
write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, csi_pv['metadata']['name'])
pod_name = 'csi-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def volume_rw_test(dev):
assert volume_valid(dev)
w_data = generate_random_data(VOLUME_RWTEST_SIZE)
l_data = len(w_data)
spos_data = generate_random_pos(VOLUME_RWTEST_SIZE)
volume_write(dev, spos_data, w_data)
r_data = volume_read(dev, spos_data, l_data)
assert r_data == w_data
def test_statefulset_recurring_backup(
client,
core_api,
storage_class, # NOQA
statefulset): # NOQA
"""
Test that recurring backups on StatefulSets work properly.
1. Create a StatefulSet with VolumeClaimTemplate and Longhorn.
2. Wait for pods to run.
3. Write some data to every pod
4. Schedule recurring jobs for volumes using Longhorn API
5. Wait for 5 minutes
6. Verify the snapshots created by the recurring jobs.
"""
statefulset_name = 'statefulset-backup-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
# backup every minute
job_backup = {
"name": "backup",
"cron": "* * * * *",
"task": "backup",
"retain": 2
}
pod_data = get_statefulset_pod_info(core_api, statefulset)
for pod in pod_data:
pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
pod['backup_snapshot'] = ''
for pod in pod_data:
volume = client.by_id_volume(pod['pv_name'])
write_pod_volume_data(core_api, pod['pod_name'], pod['data'])
volume.recurringUpdate(jobs=[job_backup])
time.sleep(150)
for pod in pod_data:
volume = client.by_id_volume(pod['pv_name'])
write_pod_volume_data(core_api, pod['pod_name'], pod['data'])
volume.recurringUpdate(jobs=[job_backup])
time.sleep(150)
for pod in pod_data:
volume = client.by_id_volume(pod['pv_name'])
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot.removed is False:
count += 1
# one backups + volume-head
assert count == 2
def snapshot_revert_test(dev, address, engine_name): # NOQA
existings = {}
snap1 = Snapshot(dev, generate_random_data(existings), address)
snap2 = Snapshot(dev, generate_random_data(existings), address)
snap3 = Snapshot(dev, generate_random_data(existings), address)
snapList = cmd.snapshot_ls(address)
assert snap1.name in snapList
assert snap2.name in snapList
assert snap3.name in snapList
snapshot_revert_with_frontend(address, engine_name, snap2.name)
snap3.refute_data()
snap2.verify_checksum()
snap1.verify_data()
snapshot_revert_with_frontend(address, engine_name, snap1.name)
snap3.refute_data()
snap2.refute_data()
snap1.verify_checksum()
def test_snapshot_revert(dev): # NOQA
existings = {}
snap1 = Snapshot(dev, generate_random_data(existings))
snap2 = Snapshot(dev, generate_random_data(existings))
snap3 = Snapshot(dev, generate_random_data(existings))
snapList = cmd.snapshot_ls()
assert snap1.name in snapList
assert snap2.name in snapList
assert snap3.name in snapList
cmd.snapshot_revert(snap2.name)
snap3.refute_data()
snap2.verify_checksum()
snap1.verify_data()
cmd.snapshot_revert(snap1.name)
snap3.refute_data()
snap2.refute_data()
snap1.verify_checksum()
def csi_backup_test(client, core_api, csi_pv, pvc, pod_make, backing_image=""): # NOQA
pod_name = 'csi-backup-test'
vol_name = create_and_wait_csi_pod(
pod_name, client, core_api, csi_pv, pvc, pod_make, backing_image, "")
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name,
vol_name, backing_image, test_data)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, vol_name)
delete_and_wait_pv(core_api, vol_name)
def test_snapshot_rm_basic(
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
existings = {}
snap1 = Snapshot(dev, generate_random_data(existings), address)
snap2 = Snapshot(dev, generate_random_data(existings), address)
snap3 = Snapshot(dev, generate_random_data(existings), address)
info = cmd.snapshot_info(address)
assert len(info) == 4
assert VOLUME_HEAD in info
assert snap1.name in info
assert snap2.name in info
assert snap3.name in info
cmd.snapshot_rm(address, snap2.name)
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
info = cmd.snapshot_info(address)
assert len(info) == 3
assert snap1.name in info
assert snap3.name in info
snap3.verify_checksum()
snap2.verify_data()
snap1.verify_data()
snapshot_revert_with_frontend(address, ENGINE_NAME, snap1.name)
snap3.refute_data()
snap2.refute_data()
snap1.verify_checksum()
def delete_data(k8s_api_client, pod_name):
file_name = 'data-' + pod_name + '.bin'
test_data = generate_random_data(0)
write_pod_volume_data(k8s_api_client,
pod_name,
test_data,
filename=file_name)
volume_data = read_volume_data(k8s_api_client,
pod_name,
filename=file_name)
assert volume_data == ""
def test_statefulset_recurring_backup(
client,
core_api,
storage_class, # NOQA
statefulset): # NOQA
"""
Test that recurring backups on StatefulSets work properly.
"""
statefulset_name = 'statefulset-backup-test'
update_test_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
# backup every minute
job_backup = {
"name": "backup",
"cron": "* * * * *",
"task": "backup",
"retain": 2
}
pod_data = get_statefulset_pod_info(core_api, statefulset)
for pod in pod_data:
pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
pod['backup_snapshot'] = ''
for pod in pod_data:
volume = client.by_id_volume(pod['pv_name'])
write_volume_data(core_api, pod['pod_name'], pod['data'])
volume.recurringUpdate(jobs=[job_backup])
time.sleep(300)
for pod in pod_data:
volume = client.by_id_volume(pod['pv_name'])
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot['removed'] is False:
count += 1
# two backups + volume-head
assert count == 3
def flexvolume_io_test(client, core_api, flexvolume, pod): # NOQA
pod_name = 'flexvolume-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \
flexvolume['name']
pod['spec']['volumes'] = [flexvolume]
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_and_wait_pod(core_api, pod)
common.write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
wait_for_volume_detached(client, flexvolume["name"])
pod_name = 'volume-driver-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def flexvolume_io_test(client, core_api, flexvolume, pod): # NOQA
pod_name = 'flexvolume-io-test'
pod['metadata']['name'] = pod_name
pod['spec']['containers'][0]['volumeMounts'][0]['name'] = \
flexvolume['name']
pod['spec']['volumes'] = [
flexvolume
]
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
create_and_wait_pod(core_api, pod)
common.write_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
wait_for_volume_detached(client, flexvolume["name"])
pod_name = 'volume-driver-io-test-2'
pod['metadata']['name'] = pod_name
create_and_wait_pod(core_api, pod)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def test_statefulset_recurring_backup(client, core_api, storage_class, # NOQA
statefulset): # NOQA
"""
Test that recurring backups on StatefulSets work properly.
"""
statefulset_name = 'statefulset-backup-test'
update_statefulset_manifests(statefulset, storage_class, statefulset_name)
create_storage_class(storage_class)
create_and_wait_statefulset(statefulset)
# backup every minute
job_backup = {"name": "backup", "cron": "* * * * *",
"task": "backup", "retain": 2}
pod_data = get_statefulset_pod_info(core_api, statefulset)
for pod in pod_data:
pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
pod['backup_snapshot'] = ''
for pod in pod_data:
volume = client.by_id_volume(pod['pv_name'])
write_volume_data(core_api, pod['pod_name'], pod['data'])
volume.recurringUpdate(jobs=[job_backup])
time.sleep(300)
for pod in pod_data:
volume = client.by_id_volume(pod['pv_name'])
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot['removed'] is False:
count += 1
# two backups + volume-head
assert count == 3
def test_xfs_pv(client, core_api, pod_manifest): # NOQA
"""
Test create PV with new XFS filesystem
1. Create a volume
2. Create a PV for the existing volume, specify `xfs` as filesystem
3. Create PVC and Pod
4. Make sure Pod is running.
5. Write data into the pod and read back for validation.
Note: The volume will be formatted to XFS filesystem by Kubernetes in this
case.
"""
volume_name = generate_volume_name()
volume = create_and_check_volume(client, volume_name)
create_pv_for_volume(client, core_api, volume, volume_name, "xfs")
create_pvc_for_volume(client, core_api, volume, volume_name)
pod_manifest['spec']['volumes'] = [{
"name": "pod-data",
"persistentVolumeClaim": {
"claimName": volume_name
}
}]
pod_name = pod_manifest['metadata']['name']
create_and_wait_pod(core_api, pod_manifest)
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def create_and_test_backups(api, cli, pod_info):
"""
Create backups for all Pods in a StatefulSet and tests that all the backups
have the correct attributes.
Args:
api: An instance of CoreV1Api.
cli: A Longhorn client instance.
pod_info: A List of Pods with names and volume information. This List
can be generated using the get_statefulset_pod_info function
located in common.py.
"""
for pod in pod_info:
pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
pod['backup_snapshot'] = ''
# Create backup.
volume = cli.by_id_volume(pod['pv_name'])
volume.snapshotCreate()
write_volume_data(api, pod['pod_name'], pod['data'])
pod['backup_snapshot'] = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=pod['backup_snapshot']['name'])
# Wait for backup to appear.
found = False
for i in range(DEFAULT_BACKUP_TIMEOUT):
backup_volumes = cli.list_backupVolume()
for bv in backup_volumes:
if bv['name'] == pod['pv_name']:
found = True
break
if found:
break
time.sleep(DEFAULT_POD_INTERVAL)
assert found
found = False
for i in range(DEFAULT_BACKUP_TIMEOUT):
backups = bv.backupList()
for b in backups:
if b['snapshotName'] == pod['backup_snapshot']['name']:
found = True
break
if found:
break
time.sleep(DEFAULT_POD_INTERVAL)
assert found
# Make sure backup has the correct attributes.
new_b = bv.backupGet(name=b["name"])
assert new_b["name"] == b["name"]
assert new_b["url"] == b["url"]
assert new_b["snapshotName"] == b["snapshotName"]
assert new_b["snapshotCreated"] == b["snapshotCreated"]
assert new_b["created"] == b["created"]
assert new_b["volumeName"] == b["volumeName"]
assert new_b["volumeSize"] == b["volumeSize"]
assert new_b["volumeCreated"] == b["volumeCreated"]
# This backup has the url attribute we need to restore from backup.
pod['backup_snapshot'] = b
def create_and_test_backups(api, cli, pod_info):
"""
Create backups for all Pods in a StatefulSet and tests that all the backups
have the correct attributes.
Args:
api: An instance of CoreV1Api.
cli: A Longhorn client instance.
pod_info: A List of Pods with names and volume information. This List
can be generated using the get_statefulset_pod_info function
located in common.py.
"""
for pod in pod_info:
pod['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
pod['backup_snapshot'] = ''
# Create backup.
volume_name = pod['pv_name']
volume = cli.by_id_volume(pod['pv_name'])
create_snapshot(cli, volume_name)
write_pod_volume_data(api, pod['pod_name'], pod['data'])
pod['backup_snapshot'] = create_snapshot(cli, volume_name)
create_snapshot(cli, volume_name)
volume.snapshotBackup(name=pod['backup_snapshot']['name'])
# Wait for backup to appear.
found = False
for i in range(DEFAULT_BACKUP_TIMEOUT):
backup_volumes = cli.list_backupVolume()
for bv in backup_volumes:
if bv.name == pod['pv_name']:
found = True
break
if found:
break
time.sleep(DEFAULT_POD_INTERVAL)
assert found
found = False
for i in range(DEFAULT_BACKUP_TIMEOUT):
backups = bv.backupList().data
for b in backups:
if b['snapshotName'] == pod['backup_snapshot']['name']:
found = True
break
if found:
break
time.sleep(DEFAULT_POD_INTERVAL)
assert found
# Make sure backup has the correct attributes.
new_b = bv.backupGet(name=b.name)
assert new_b.name == b.name
assert new_b.url == b.url
assert new_b.snapshotName == b.snapshotName
assert new_b.snapshotCreated == b.snapshotCreated
assert new_b.created == b.created
assert new_b.volumeName == b.volumeName
assert new_b.volumeSize == b.volumeSize
assert new_b.volumeCreated == b.volumeCreated
# This backup has the url attribute we need to restore from backup.
pod['backup_snapshot'] = b
def test_rwx_multi_statefulset_with_same_pvc(core_api, pvc, statefulset,
pod): # NOQA
"""
Test writing of data into a volume from multiple pods using same PVC
1. Create a volume with 'accessMode' rwx.
2. Create a PV and a PVC with access mode 'readwritemany' and attach to the
volume.
3. Deploy a StatefulSet of 2 pods with the existing PVC above created.
4. Wait for both pods to come up.
5. Create a pod with the existing PVC above created.
6. Wait for StatefulSet to come up healthy.
7. Write data all three pods and compute md5sum.
8. Check the data md5sum in the share manager pod.
"""
pvc_name = 'pvc-multi-pods-test'
statefulset_name = 'statefulset-rwx-same-pvc-test'
pod_name = 'pod-rwx-same-pvc-test'
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = 'longhorn'
pvc['spec']['accessModes'] = ['ReadWriteMany']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
statefulset['metadata']['name'] = \
statefulset['spec']['selector']['matchLabels']['app'] = \
statefulset['spec']['serviceName'] = \
statefulset['spec']['template']['metadata']['labels']['app'] = \
statefulset_name
statefulset['spec']['template']['spec']['volumes'] = \
[create_pvc_spec(pvc_name)]
del statefulset['spec']['volumeClaimTemplates']
create_and_wait_statefulset(statefulset)
pv_name = get_volume_name(core_api, pvc_name)
share_manager_name = 'share-manager-' + pv_name
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api,
statefulset_name + '-0',
test_data,
filename='test1')
assert test_data == read_volume_data(core_api,
statefulset_name + '-1',
filename='test1')
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
create_and_wait_pod(core_api, pod)
assert test_data == read_volume_data(core_api, pod_name, filename='test1')
test_data_2 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data_2, filename='test2')
command1 = 'cat /export' + '/' + pv_name + '/' + 'test1'
command2 = 'cat /export' + '/' + pv_name + '/' + 'test2'
assert test_data == exec_command_in_pod(core_api, command1,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_2 == exec_command_in_pod(core_api, command2,
share_manager_name,
LONGHORN_NAMESPACE)
def test_rwx_deployment_with_multi_pods(core_api, pvc,
make_deployment_with_pvc): # NOQA
"""
Test deployment of 2 pods with same PVC.
1. Create a volume with 'accessMode' rwx.
2. Create a PV and a PVC with access mode 'readwritemany' and attach to the
volume.
3. Create a deployment of 2 pods with PVC created
4. Wait for 2 pods to come up healthy.
5. Write data in both pods and compute md5sum.
6. Check the data md5sum in the share manager pod.
"""
pvc_name = 'pvc-deployment-multi-pods-test'
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = 'longhorn'
pvc['spec']['accessModes'] = ['ReadWriteMany']
core_api.create_namespaced_persistent_volume_claim(body=pvc,
namespace='default')
deployment = make_deployment_with_pvc('deployment-multi-pods-test',
pvc_name,
replicas=2)
apps_api = get_apps_api_client()
create_and_wait_deployment(apps_api, deployment)
pv_name = get_volume_name(core_api, pvc_name)
share_manager_name = 'share-manager-' + pv_name
deployment_label_selector = "name=" + \
deployment["metadata"]["labels"]["name"]
deployment_pod_list = \
core_api.list_namespaced_pod(namespace="default",
label_selector=deployment_label_selector)
assert deployment_pod_list.items.__len__() == 2
pod_name_1 = deployment_pod_list.items[0].metadata.name
test_data_1 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name_1, test_data_1, filename='test1')
pod_name_2 = deployment_pod_list.items[1].metadata.name
command = 'cat /data/test1'
pod_data_2 = exec_command_in_pod(core_api, command, pod_name_2, 'default')
assert test_data_1 == pod_data_2
test_data_2 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name_2, test_data_2, filename='test2')
command = 'cat /export' + '/' + pv_name + '/' + 'test1'
share_manager_data_1 = exec_command_in_pod(core_api, command,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_1 == share_manager_data_1
command = 'cat /export' + '/' + pv_name + '/' + 'test2'
share_manager_data_2 = exec_command_in_pod(core_api, command,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_2 == share_manager_data_2
def test_rwx_delete_share_manager_pod(core_api, statefulset): # NOQA
"""
Test moving of Share manager pod from one node to another.
1. Create a StatefulSet of 1 pod with VolumeClaimTemplate where accessMode
is 'RWX'.
2. Wait for StatefulSet to come up healthy.
3. Write data and compute md5sum.
4. Delete the share manager pod.
5. Wait for a new pod to be created and volume getting attached.
6. Check the data md5sum in statefulSet.
7. Write more data to it and compute md5sum.
8. Check the data md5sum in share manager volume.
"""
statefulset_name = 'statefulset-delete-share-manager-pods-test'
statefulset['metadata']['name'] = \
statefulset['spec']['selector']['matchLabels']['app'] = \
statefulset['spec']['serviceName'] = \
statefulset['spec']['template']['metadata']['labels']['app'] = \
statefulset_name
statefulset['spec']['replicas'] = 1
statefulset['spec']['volumeClaimTemplates'][0]['spec']['storageClassName']\
= 'longhorn'
statefulset['spec']['volumeClaimTemplates'][0]['spec']['accessModes'] \
= ['ReadWriteMany']
create_and_wait_statefulset(statefulset)
pod_name = statefulset_name + '-' + '0'
pvc_name = \
statefulset['spec']['volumeClaimTemplates'][0]['metadata']['name'] \
+ '-' + statefulset_name + '-0'
pv_name = get_volume_name(core_api, pvc_name)
share_manager_name = 'share-manager-' + pv_name
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data, filename='test1')
delete_and_wait_pod(core_api,
share_manager_name,
namespace=LONGHORN_NAMESPACE)
target_pod = core_api.read_namespaced_pod(name=pod_name,
namespace='default')
wait_delete_pod(core_api, target_pod.metadata.uid)
wait_for_pod_remount(core_api, pod_name)
test_data_2 = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data_2, filename='test2')
command1 = 'cat /export/' + pv_name + '/test1'
share_manager_data_1 = exec_command_in_pod(core_api, command1,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data == share_manager_data_1
command2 = 'cat /export/' + pv_name + '/test2'
share_manager_data_2 = exec_command_in_pod(core_api, command2,
share_manager_name,
LONGHORN_NAMESPACE)
assert test_data_2 == share_manager_data_2
def test_rwx_with_statefulset_multi_pods(core_api, statefulset): # NOQA
"""
Test creation of share manager pod and rwx volumes from 2 pods.
1. Create a StatefulSet of 2 pods with VolumeClaimTemplate where accessMode
is 'RWX'.
2. Wait for both pods to come up running.
3. Verify there are two share manager pods created in the longhorn
namespace and they have the directory with the PV name in the
path `/export`
4. Write data in both pods and compute md5sum.
5. Compare md5sum of the data with the data written the share manager.
"""
statefulset_name = 'statefulset-rwx-multi-pods-test'
share_manager_name = []
volumes_name = []
statefulset['metadata']['name'] = \
statefulset['spec']['selector']['matchLabels']['app'] = \
statefulset['spec']['serviceName'] = \
statefulset['spec']['template']['metadata']['labels']['app'] = \
statefulset_name
statefulset['spec']['volumeClaimTemplates'][0]['spec']['storageClassName']\
= 'longhorn'
statefulset['spec']['volumeClaimTemplates'][0]['spec']['accessModes'] \
= ['ReadWriteMany']
create_and_wait_statefulset(statefulset)
for i in range(2):
pvc_name = \
statefulset['spec']['volumeClaimTemplates'][0]['metadata']['name']\
+ '-' + statefulset_name + '-' + str(i)
pv_name = get_volume_name(core_api, pvc_name)
assert pv_name is not None
volumes_name.append(pv_name)
share_manager_name.append('share-manager-' + pv_name)
check_pod_existence(core_api,
share_manager_name[i],
namespace=LONGHORN_NAMESPACE)
command = "ls /export | grep -i 'pvc' | wc -l"
assert exec_command_in_pod(core_api, command, share_manager_name[0],
LONGHORN_NAMESPACE) == '1'
assert exec_command_in_pod(core_api, command, share_manager_name[1],
LONGHORN_NAMESPACE) == '1'
md5sum_pod = []
for i in range(2):
test_pod_name = statefulset_name + '-' + str(i)
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, test_pod_name, test_data)
md5sum_pod.append(test_data)
for i in range(2):
command = 'cat /export' + '/' + volumes_name[i] + '/' + 'test'
pod_data = exec_command_in_pod(core_api, command,
share_manager_name[i],
LONGHORN_NAMESPACE)
assert pod_data == md5sum_pod[i]
def test_rwx_statefulset_scale_down_up(core_api, statefulset): # NOQA
"""
Test Scaling up and down of pods attached to rwx volume.
1. Create a StatefulSet of 2 pods with VolumeClaimTemplate where accessMode
is 'RWX'.
2. Wait for StatefulSet pods to come up healthy.
3. Write data and compute md5sum in the both pods.
4. Delete the pods.
5. Wait for the pods to be terminated.
6. Verify the share manager pods are no longer available and the volume is
detached.
6. Recreate the pods
7. Wait for new pods to come up.
8. Check the data md5sum in new pods.
"""
statefulset_name = 'statefulset-rwx-scale-down-up-test'
share_manager_name = []
statefulset['metadata']['name'] = \
statefulset['spec']['selector']['matchLabels']['app'] = \
statefulset['spec']['serviceName'] = \
statefulset['spec']['template']['metadata']['labels']['app'] = \
statefulset_name
statefulset['spec']['volumeClaimTemplates'][0]['spec']['storageClassName']\
= 'longhorn'
statefulset['spec']['volumeClaimTemplates'][0]['spec']['accessModes'] \
= ['ReadWriteMany']
create_and_wait_statefulset(statefulset)
for i in range(2):
pvc_name = \
statefulset['spec']['volumeClaimTemplates'][0]['metadata']['name']\
+ '-' + statefulset_name + '-' + str(i)
pv_name = get_volume_name(core_api, pvc_name)
assert pv_name is not None
share_manager_name.append('share-manager-' + pv_name)
check_pod_existence(core_api,
share_manager_name[i],
namespace=LONGHORN_NAMESPACE)
md5sum_pod = []
for i in range(2):
test_pod_name = statefulset_name + '-' + str(i)
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, test_pod_name, test_data)
md5sum_pod.append(test_data)
statefulset['spec']['replicas'] = replicas = 0
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={'spec': {
'replicas': replicas
}})
for i in range(DEFAULT_STATEFULSET_TIMEOUT):
s_set = apps_api.read_namespaced_stateful_set(
name=statefulset['metadata']['name'], namespace='default')
if s_set.status.ready_replicas == replicas or \
(replicas == 0 and not s_set.status.ready_replicas):
break
time.sleep(DEFAULT_STATEFULSET_INTERVAL)
pods = core_api.list_namespaced_pod(namespace=LONGHORN_NAMESPACE)
found = False
for item in pods.items:
if item.metadata.name == share_manager_name[0] or \
item.metadata.name == share_manager_name[1]:
found = True
break
assert not found
statefulset['spec']['replicas'] = replicas = 2
apps_api = get_apps_api_client()
apps_api.patch_namespaced_stateful_set(
name=statefulset_name,
namespace='default',
body={'spec': {
'replicas': replicas
}})
wait_statefulset(statefulset)
for i in range(2):
test_pod_name = statefulset_name + '-' + str(i)
command = 'cat /data/test'
pod_data = exec_command_in_pod(core_api, command, test_pod_name,
'default')
assert pod_data == md5sum_pod[i]
def test_csi_block_volume(client, core_api, storage_class, pvc, pod_manifest): # NOQA
"""
Test CSI feature: raw block volume
1. Create a PVC with `volumeMode = Block`
2. Create a pod using the PVC to dynamic provision a volume
3. Verify the pod creation
4. Generate `test_data` and write to the block volume directly in the pod
5. Read the data back for validation
6. Delete the pod and create `pod2` to use the same volume
7. Validate the data in `pod2` is consistent with `test_data`
"""
pod_name = 'csi-block-volume-test'
pvc_name = pod_name + "-pvc"
device_path = "/dev/longhorn/longhorn-test-blk"
storage_class['reclaimPolicy'] = 'Retain'
pvc['metadata']['name'] = pvc_name
pvc['spec']['volumeMode'] = 'Block'
pvc['spec']['storageClassName'] = storage_class['metadata']['name']
pvc['spec']['resources'] = {
'requests': {
'storage': size_to_string(1 * Gi)
}
}
pod_manifest['metadata']['name'] = pod_name
pod_manifest['spec']['volumes'] = [{
'name': 'longhorn-blk',
'persistentVolumeClaim': {
'claimName': pvc_name,
},
}]
pod_manifest['spec']['containers'][0]['volumeMounts'] = []
pod_manifest['spec']['containers'][0]['volumeDevices'] = [
{'name': 'longhorn-blk', 'devicePath': device_path}
]
create_storage_class(storage_class)
create_pvc(pvc)
pv_name = wait_and_get_pv_for_pvc(core_api, pvc_name).metadata.name
create_and_wait_pod(core_api, pod_manifest)
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
test_offset = random.randint(0, VOLUME_RWTEST_SIZE)
write_pod_block_volume_data(
core_api, pod_name, test_data, test_offset, device_path)
returned_data = read_pod_block_volume_data(
core_api, pod_name, len(test_data), test_offset, device_path
)
assert test_data == returned_data
md5_sum = get_pod_data_md5sum(
core_api, pod_name, device_path)
delete_and_wait_pod(core_api, pod_name)
common.wait_for_volume_detached(client, pv_name)
pod_name_2 = 'csi-block-volume-test-reuse'
pod_manifest['metadata']['name'] = pod_name_2
create_and_wait_pod(core_api, pod_manifest)
returned_data = read_pod_block_volume_data(
core_api, pod_name_2, len(test_data), test_offset, device_path
)
assert test_data == returned_data
md5_sum_2 = get_pod_data_md5sum(
core_api, pod_name_2, device_path)
assert md5_sum == md5_sum_2
delete_and_wait_pod(core_api, pod_name_2)
delete_and_wait_pvc(core_api, pvc_name)
delete_and_wait_pv(core_api, pv_name)
def test_csi_offline_expansion(client, core_api, storage_class, pvc, pod_manifest): # NOQA
"""
Test CSI feature: offline expansion
1. Create a new `storage_class` with `allowVolumeExpansion` set
2. Create PVC and Pod with dynamic provisioned volume from the StorageClass
3. Generate `test_data` and write to the pod
4. Delete the pod
5. Update pvc.spec.resources to expand the volume
6. Verify the volume expansion done using Longhorn API
7. Create a new pod and validate the volume content
"""
create_storage_class(storage_class)
pod_name = 'csi-offline-expand-volume-test'
pvc_name = pod_name + "-pvc"
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = storage_class['metadata']['name']
create_pvc(pvc)
pod_manifest['metadata']['name'] = pod_name
pod_manifest['spec']['volumes'] = [{
'name':
pod_manifest['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {'claimName': pvc_name},
}]
create_and_wait_pod(core_api, pod_manifest)
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
pv = wait_and_get_pv_for_pvc(core_api, pvc_name)
assert pv.status.phase == "Bound"
volume_name = pv.spec.csi.volume_handle
wait_for_volume_detached(client, volume_name)
pvc['spec']['resources'] = {
'requests': {
'storage': size_to_string(EXPANDED_VOLUME_SIZE*Gi)
}
}
expand_and_wait_for_pvc(core_api, pvc)
wait_for_volume_expansion(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume.state == "detached"
assert volume.size == str(EXPANDED_VOLUME_SIZE*Gi)
pod_manifest['metadata']['name'] = pod_name
pod_manifest['spec']['volumes'] = [{
'name':
pod_manifest['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {'claimName': pvc_name},
}]
create_and_wait_pod(core_api, pod_manifest)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
volume = client.by_id_volume(volume_name)
engine = get_volume_engine(volume)
assert volume.size == str(EXPANDED_VOLUME_SIZE*Gi)
assert volume.size == engine.size
def test_offline_node_with_attached_volume_and_pod(
client, core_api, volume_name, make_deployment_with_pvc,
reset_cluster_ready_status): # NOQA
"""
Test offline node with attached volume and pod
1. Create PV/PVC/Deployment manifest.
2. Update deployment's tolerations to 20 seconds to speed up test
3. Update deployment's node affinity rule to avoid the current node
4. Create volume, PV/PVC and deployment.
5. Find the pod in the deployment and write `test_data` into it
6. Shutdown the node pod is running on
7. Wait for deployment to delete the pod
1. Deployment cannot delete the pod here because kubelet doesn't
response
8. Force delete the terminating pod
9. Wait for the new pod to be created and the volume attached
10. Check `test_data` in the new pod
"""
toleration_seconds = 20
apps_api = get_apps_api_client()
cloudprovider = detect_cloudprovider()
volume_name = generate_volume_name()
pv_name = volume_name + "-pv"
pvc_name = volume_name + "-pvc"
deployment_name = volume_name + "-dep"
longhorn_test_node_name = get_self_host_id()
deployment_manifest = make_deployment_with_pvc(deployment_name, pvc_name)
unreachable_toleration = {
"key": "node.kubernetes.io/unreachable",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": toleration_seconds
}
not_ready_toleration = {
"key": "node.kubernetes.io/not-ready",
"operator": "Exists",
"effect": "NoExecute",
"tolerationSeconds": toleration_seconds
}
deployment_manifest["spec"]["template"]["spec"]["tolerations"] =\
[unreachable_toleration, not_ready_toleration]
node_affinity_roles = {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kubernetes.io/hostname",
"operator": "NotIn",
"values": [longhorn_test_node_name]
}]
}]
}
}
}
deployment_manifest["spec"]["template"]["spec"]["affinity"] =\
node_affinity_roles
longhorn_volume = create_and_check_volume(client, volume_name, size=SIZE)
wait_for_volume_detached(client, volume_name)
create_pv_for_volume(client, core_api, longhorn_volume, pv_name)
create_pvc_for_volume(client, core_api, longhorn_volume, pvc_name)
create_and_wait_deployment(apps_api, deployment_manifest)
deployment_label_selector =\
"name=" + deployment_manifest["metadata"]["labels"]["name"]
deployment_pod_list =\
core_api.list_namespaced_pod(namespace="default",
label_selector=deployment_label_selector)
assert deployment_pod_list.items.__len__() == 1
pod_name = deployment_pod_list.items[0].metadata.name
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
node_name = deployment_pod_list.items[0].spec.node_name
node = cloudprovider.node_id(node_name)
cloudprovider.node_shutdown(node)
k8s_node_down = wait_for_node_down_k8s(node_name, core_api)
assert k8s_node_down
client = get_longhorn_api_client()
longhorn_node_down = wait_for_node_down_longhorn(node_name, client)
assert longhorn_node_down
time.sleep(toleration_seconds + 5)
for i in range(TERMINATING_POD_RETRYS):
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
terminating_pod_name = None
for pod in deployment_pod_list.items:
if pod.metadata.__getattribute__("deletion_timestamp") is not None:
terminating_pod_name = pod.metadata.name
break
if terminating_pod_name is not None:
break
else:
time.sleep(TERMINATING_POD_INTERVAL)
assert terminating_pod_name is not None
core_api.delete_namespaced_pod(namespace="default",
name=terminating_pod_name,
grace_period_seconds=0)
delete_and_wait_pod(core_api, terminating_pod_name)
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
assert deployment_pod_list.items.__len__() == 1
wait_for_volume_detached(client, volume_name)
wait_for_volume_healthy(client, volume_name)
deployment_pod_list =\
core_api.list_namespaced_pod(
namespace="default",
label_selector=deployment_label_selector
)
assert deployment_pod_list.items.__len__() == 1
new_pod_name = deployment_pod_list.items[0].metadata.name
wait_pod(new_pod_name)
resp_data = read_volume_data(core_api, new_pod_name)
assert test_data == resp_data
def test_csi_expansion_with_replica_failure(client, core_api, storage_class, pvc, pod_manifest): # NOQA
"""
Test expansion success but with one replica expansion failure
1. Create a new `storage_class` with `allowVolumeExpansion` set
2. Create PVC and Pod with dynamic provisioned volume from the StorageClass
3. Create an empty directory with expansion snapshot tmp meta file path
for one replica so that the replica expansion will fail
4. Generate `test_data` and write to the pod
5. Delete the pod and wait for volume detachment
6. Update pvc.spec.resources to expand the volume
7. Check expansion result using Longhorn API. There will be expansion error
caused by the failed replica but overall the expansion should succeed.
8. Create a new pod and
check if the volume will rebuild the failed replica
9. Validate the volume content, then check if data writing looks fine
"""
create_storage_class(storage_class)
pod_name = 'csi-expansion-with-replica-failure-test'
pvc_name = pod_name + "-pvc"
pvc['metadata']['name'] = pvc_name
pvc['spec']['storageClassName'] = storage_class['metadata']['name']
create_pvc(pvc)
pod_manifest['metadata']['name'] = pod_name
pod_manifest['spec']['volumes'] = [{
'name':
pod_manifest['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {'claimName': pvc_name},
}]
create_and_wait_pod(core_api, pod_manifest)
expand_size = str(EXPANDED_VOLUME_SIZE*Gi)
pv = wait_and_get_pv_for_pvc(core_api, pvc_name)
assert pv.status.phase == "Bound"
volume_name = pv.spec.csi.volume_handle
volume = client.by_id_volume(volume_name)
failed_replica = volume.replicas[0]
fail_replica_expansion(client, core_api,
volume_name, expand_size, [failed_replica])
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
delete_and_wait_pod(core_api, pod_name)
wait_for_volume_detached(client, volume_name)
# There will be replica expansion error info
# but the expansion should succeed.
pvc['spec']['resources'] = {
'requests': {
'storage': size_to_string(EXPANDED_VOLUME_SIZE*Gi)
}
}
expand_and_wait_for_pvc(core_api, pvc)
wait_for_expansion_failure(client, volume_name)
wait_for_volume_expansion(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume.state == "detached"
assert volume.size == expand_size
for r in volume.replicas:
if r.name == failed_replica.name:
assert r.failedAt != ""
else:
assert r.failedAt == ""
# Check if the replica will be rebuilded
# and if the volume still works fine.
create_and_wait_pod(core_api, pod_manifest)
volume = wait_for_volume_healthy(client, volume_name)
for r in volume.replicas:
if r.name == failed_replica.name:
assert r.mode == ""
else:
assert r.mode == "RW"
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
test_data = generate_random_data(VOLUME_RWTEST_SIZE)
write_pod_volume_data(core_api, pod_name, test_data)
resp = read_volume_data(core_api, pod_name)
assert resp == test_data
def backup_test(client, host_id, volname):
volume = client.by_id_volume(volname)
volume.snapshotCreate()
w_data = generate_random_data(VOLUME_RWTEST_SIZE)
start_pos = generate_random_pos(VOLUME_RWTEST_SIZE)
l_data = volume_write(volume["endpoint"], start_pos, w_data)
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
found = False
for i in range(100):
bvs = client.list_backupVolume()
for bv in bvs:
if bv["name"] == volname:
found = True
break
if found:
break
time.sleep(1)
assert found
found = False
for i in range(20):
backups = bv.backupList()
for b in backups:
if b["snapshotName"] == snap2["name"]:
found = True
break
if found:
break
time.sleep(1)
assert found
new_b = bv.backupGet(name=b["name"])
assert new_b["name"] == b["name"]
assert new_b["url"] == b["url"]
assert new_b["snapshotName"] == b["snapshotName"]
assert new_b["snapshotCreated"] == b["snapshotCreated"]
assert new_b["created"] == b["created"]
assert new_b["volumeName"] == b["volumeName"]
assert new_b["volumeSize"] == b["volumeSize"]
assert new_b["volumeCreated"] == b["volumeCreated"]
# test restore
restoreName = generate_volume_name()
volume = client.create_volume(name=restoreName, size=SIZE,
numberOfReplicas=2,
fromBackup=b["url"])
volume = common.wait_for_volume_detached(client, restoreName)
assert volume["name"] == restoreName
assert volume["size"] == SIZE
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, restoreName)
r_data = volume_read(volume["endpoint"], start_pos, l_data)
assert r_data == w_data
volume = volume.detach()
volume = common.wait_for_volume_detached(client, restoreName)
client.delete(volume)
volume = wait_for_volume_delete(client, restoreName)
bv.backupDelete(name=b["name"])
backups = bv.backupList()
found = False
for b in backups:
if b["snapshotName"] == snap2["name"]:
found = True
break
assert not found
