def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA
for host_id, client in clients.iteritems(): # NOQA
break
# error when creating volume with duplicate jobs
with pytest.raises(Exception) as e:
client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1() + create_jobs1())
assert "duplicate job" in str(e.value)
client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2, recurringJobs=create_jobs1())
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def ha_simple_recovery_test(client, volume_name, size, base_image=""): # NOQA
volume = client.create_volume(name=volume_name, size=size,
numberOfReplicas=2, baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == size
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["created"] != ""
assert volume["baseImage"] == base_image
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
ha_rebuild_replica_test(client, volume_name)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_replica_scheduler_no_disks(client): # NOQA
nodes = client.list_node()
# delete all disks on each node
for node in nodes:
disks = node["disks"]
name = node["name"]
# set allowScheduling to false
for fsid, disk in disks.iteritems():
disk["allowScheduling"] = False
update_disks = get_update_disks(disks)
node = node.diskUpdate(disks=update_disks)
for fsid, disk in node["disks"].iteritems():
# wait for node controller update disk status
wait_for_disk_status(client, name, fsid,
"allowScheduling", False)
wait_for_disk_status(client, name, fsid,
"storageScheduled", 0)
node = client.by_id_node(name)
for fsid, disk in node["disks"].iteritems():
assert not disk["allowScheduling"]
node = node.diskUpdate(disks=[])
node = common.wait_for_disk_update(client, name, 0)
assert len(node["disks"]) == 0
# test there's no disk fit for volume
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=SIZE, numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
client.delete(volume)
common.wait_for_volume_delete(client, vol_name)
def test_replica_scheduler_exceed_over_provisioning(client): # NOQA
over_provisioning_setting = client.by_id_setting(
SETTING_STORAGE_OVER_PROVISIONING_PERCENTAGE)
old_provisioning_setting = over_provisioning_setting["value"]
# set storage over provisioning percentage to 100
over_provisioning_setting = client.update(over_provisioning_setting,
value="100")
# test exceed over provisioning limit couldn't be scheduled
nodes = client.list_node()
for node in nodes:
disks = node["disks"]
for fsid, disk in disks.iteritems():
disk["storageReserved"] = \
disk["storageMaximum"] - 1*Gi
update_disks = get_update_disks(disks)
node = node.diskUpdate(disks=update_disks)
disks = node["disks"]
for fsid, disk in disks.iteritems():
wait_for_disk_status(client, node["name"],
fsid, "storageReserved",
disk["storageMaximum"] - 1*Gi)
vol_name = common.generate_volume_name()
volume = client.create_volume(name=vol_name,
size=str(2*Gi), numberOfReplicas=len(nodes))
volume = common.wait_for_volume_condition_scheduled(client, vol_name,
"status",
CONDITION_STATUS_FALSE)
client.delete(volume)
common.wait_for_volume_delete(client, vol_name)
client.update(over_provisioning_setting, value=old_provisioning_setting)
def test_ha_prohibit_deleting_last_replica(client, volume_name): # NOQA
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=1)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == SIZE
assert volume["numberOfReplicas"] == 1
assert volume["state"] == "detached"
assert volume["created"] != ""
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 1
replica0 = volume["replicas"][0]
with pytest.raises(Exception) as e:
volume.replicaRemove(name=replica0["name"])
assert "no other healthy replica available" in str(e.value)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def ha_salvage_test(client, volume_name, base_image=""): # NOQA
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == SIZE
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["created"] != ""
assert volume["baseImage"] == base_image
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 2
replica0_name = volume["replicas"][0]["name"]
replica1_name = volume["replicas"][1]["name"]
data = write_volume_random_data(volume)
common.k8s_delete_replica_pods_for_volume(volume_name)
volume = common.wait_for_volume_faulted(client, volume_name)
assert len(volume["replicas"]) == 2
assert volume["replicas"][0]["failedAt"] != ""
assert volume["replicas"][1]["failedAt"] != ""
volume.salvage(names=[replica0_name, replica1_name])
volume = common.wait_for_volume_detached(client, volume_name)
assert len(volume["replicas"]) == 2
assert volume["replicas"][0]["failedAt"] == ""
assert volume["replicas"][1]["failedAt"] == ""
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
check_volume_data(volume, data)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_csi_expansion_with_size_round_up(client, core_api): # NOQA
"""
test expand longhorn volume
1. Create longhorn volume with size '1Gi'
2. Attach, write data, and detach
3. Expand volume size to '2000000000/2G' and
check if size round up '2000683008'
4. Attach, write data, and detach
5. Expand volume size to '2Gi' and check if size is '2147483648'
6. Attach, write data, and detach
"""
volume_name = generate_volume_name()
volume = create_and_check_volume(client, volume_name, 2, str(1 * Gi))
self_hostId = get_self_host_id()
volume.attach(hostId=self_hostId, disableFrontend=False)
volume = wait_for_volume_healthy(client, volume_name)
test_data = write_volume_random_data(volume)
volume.detach(hostId="")
volume = wait_for_volume_detached(client, volume_name)
volume.expand(size="2000000000")
wait_for_volume_expansion(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume.size == "2000683008"
self_hostId = get_self_host_id()
volume.attach(hostId=self_hostId, disableFrontend=False)
volume = wait_for_volume_healthy(client, volume_name)
check_volume_data(volume, test_data, False)
test_data = write_volume_random_data(volume)
volume.detach(hostId="")
volume = wait_for_volume_detached(client, volume_name)
volume.expand(size=str(2 * Gi))
wait_for_volume_expansion(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume.size == "2147483648"
self_hostId = get_self_host_id()
volume.attach(hostId=self_hostId, disableFrontend=False)
volume = wait_for_volume_healthy(client, volume_name)
check_volume_data(volume, test_data, False)
volume.detach(hostId="")
volume = wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
def test_tag_scheduling_failure(client, node_default_tags): # NOQA
"""
Test that scheduling fails if no Nodes/Disks with the requested Tags are
available.
"""
invalid_tag_cases = [
# Only one Disk Tag exists.
{
"disk": ["doesnotexist", "ssd"],
"node": []
},
# Only one Node Tag exists.
{
"disk": [],
"node": ["doesnotexist", "main"]
}
]
for tags in invalid_tag_cases:
volume_name = generate_volume_name() # NOQA
with pytest.raises(Exception) as e:
client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=3,
diskSelector=tags["disk"],
nodeSelector=tags["node"])
assert "does not exist" in str(e.value)
unsatisfied_tag_cases = [
{
"disk": [],
"node": ["main", "fallback"]
},
{
"disk": ["ssd", "m2"],
"node": []
}
]
for tags in unsatisfied_tag_cases:
volume_name = generate_volume_name()
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=3,
diskSelector=tags["disk"],
nodeSelector=tags["node"])
volume = wait_for_volume_detached(client, volume_name)
assert volume["diskSelector"] == tags["disk"]
assert volume["nodeSelector"] == tags["node"]
wait_scheduling_failure(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume["name"])
volumes = client.list_volume()
assert len(volumes) == 0
def test_recurring_job(clients, volume_name): # NOQA
for host_id, client in clients.iteritems(): # NOQA
break
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2)
volume = common.wait_for_volume_detached(client, volume_name)
jobs = create_jobs1()
volume.recurringUpdate(jobs=jobs)
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
job_backup2 = {
"name": "backup2",
"cron": "* * * * *",
"task": "backup",
"retain": 2
}
volume.recurringUpdate(jobs=[jobs[0], job_backup2])
# 5 minutes
time.sleep(300)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot["removed"] is False:
count += 1
# 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head
assert count == 6
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def ha_salvage_test(client, volume_name, base_image=""): # NOQA
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2, baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == SIZE
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["created"] != ""
assert volume["baseImage"] == base_image
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert len(volume["replicas"]) == 2
replica0_name = volume["replicas"][0]["name"]
replica1_name = volume["replicas"][1]["name"]
data = write_volume_random_data(volume)
common.k8s_delete_replica_pods_for_volume(volume_name)
volume = common.wait_for_volume_faulted(client, volume_name)
assert len(volume["replicas"]) == 2
assert volume["replicas"][0]["failedAt"] != ""
assert volume["replicas"][1]["failedAt"] != ""
volume.salvage(names=[replica0_name, replica1_name])
volume = common.wait_for_volume_detached(client, volume_name)
assert len(volume["replicas"]) == 2
assert volume["replicas"][0]["failedAt"] == ""
assert volume["replicas"][1]["failedAt"] == ""
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
check_volume_data(volume, data)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_recurring_job_in_volume_creation(set_random_backupstore, client,
volume_name): # NOQA
"""
Test create volume with recurring jobs
1. Create volume with recurring jobs though Longhorn API
2. Verify the recurring jobs run correctly
"""
host_id = get_self_host_id()
# error when creating volume with duplicate jobs
with pytest.raises(Exception) as e:
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1() + create_jobs1())
assert "duplicate job" in str(e.value)
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1())
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
# wait until the beginning of an even minute
wait_until_begin_of_an_even_minute()
# wait until the 10th second of an even minute
# to avoid writing data at the same time backup is taking
time.sleep(10)
write_volume_random_data(volume)
time.sleep(150) # 2.5 minutes
write_volume_random_data(volume)
time.sleep(150) # 2.5 minutes
check_jobs1_result(volume)
volume = volume.detach(hostId="")
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_recurring_job(clients, volume_name): # NOQA
for host_id, client in clients.iteritems(): # NOQA
break
volume = client.create_volume(name=volume_name, size=SIZE,
numberOfReplicas=2)
volume = common.wait_for_volume_detached(client, volume_name)
jobs = create_jobs1()
volume.recurringUpdate(jobs=jobs)
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
job_backup2 = {"name": "backup2", "cron": "* * * * *",
"task": "backup", "retain": 2}
volume.recurringUpdate(jobs=[jobs[0], job_backup2])
# 5 minutes
time.sleep(300)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot["removed"] is False:
count += 1
# 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head
assert count == 6
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_recurring_job_in_volume_creation(clients, volume_name): # NOQA
"""
Test create volume with recurring jobs
1. Create volume with recurring jobs though Longhorn API
2. Verify the recurring jobs run correctly
"""
for host_id, client in iter(clients.items()): # NOQA
break
set_random_backupstore(client)
# error when creating volume with duplicate jobs
with pytest.raises(Exception) as e:
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1() + create_jobs1())
assert "duplicate job" in str(e.value)
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
recurringJobs=create_jobs1())
volume = common.wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_engine_image_incompatible(client, core_api, volume_name): # NOQA
"""
Test incompatible engine images
1. Deploy incompatible engine images
2. Make sure their state are `incompatible` once deployed.
"""
images = client.list_engine_image()
assert len(images) == 1
assert images[0].default
assert images[0].state == "ready"
cli_v = images[0].cliAPIVersion
# cli_minv = images[0].cliAPIMinVersion
ctl_v = images[0].controllerAPIVersion
ctl_minv = images[0].controllerAPIMinVersion
data_v = images[0].dataFormatVersion
data_minv = images[0].dataFormatMinVersion
fail_cli_v_image = common.get_compatibility_test_image(
cli_v - 1, cli_v - 1, ctl_v, ctl_minv, data_v, data_minv)
img = client.create_engine_image(image=fail_cli_v_image)
img_name = img.name
img = wait_for_engine_image_state(client, img_name, "incompatible")
assert img.state == "incompatible"
assert img.cliAPIVersion == cli_v - 1
assert img.cliAPIMinVersion == cli_v - 1
client.delete(img)
wait_for_engine_image_deletion(client, core_api, img.name)
fail_cli_minv_image = common.get_compatibility_test_image(
cli_v + 1, cli_v + 1, ctl_v, ctl_minv, data_v, data_minv)
img = client.create_engine_image(image=fail_cli_minv_image)
img_name = img.name
img = wait_for_engine_image_state(client, img_name, "incompatible")
assert img.state == "incompatible"
assert img.cliAPIVersion == cli_v + 1
assert img.cliAPIMinVersion == cli_v + 1
client.delete(img)
wait_for_engine_image_deletion(client, core_api, img.name)
def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, vol_name, backing_image, test_data): # NOQA
write_pod_volume_data(core_api, pod_name, test_data)
volume = client.by_id_volume(vol_name)
snap = create_snapshot(client, vol_name)
volume.snapshotBackup(name=snap.name)
common.wait_for_backup_completion(client, vol_name, snap.name)
bv, b = common.find_backup(client, vol_name, snap.name)
pod2_name = 'csi-backup-test-2'
vol2_name = create_and_wait_csi_pod(
pod2_name, client, core_api, csi_pv, pvc, pod_make,
backing_image, b.url)
volume2 = client.by_id_volume(vol2_name)
resp = read_volume_data(core_api, pod2_name)
assert resp == test_data
delete_backup(client, bv.name, b.name)
delete_and_wait_pod(core_api, pod2_name)
client.delete(volume2)
def test_delete_with_provisioned_pv(client, core_api, storage_class, pvc): # NOQA
"""
Test that deleting a Volume with dynamically provisioned Persistent Volume
and Persistent Volume Claim resources successfully deletes the Volume and
cleans up those resources.
1. Create a Storage Class to test with.
2. Create a Persistent Volume Claim that requests a Volume from that
Storage Class.
3. Wait for the Volume to be provisioned and for the Kubernetes Status to
be updated correctly.
4. Attempt to delete the Volume.
5. Verify that the Volume and its associated resources have been deleted.
"""
pv = provision_and_wait_pv(client, core_api, storage_class, pvc)
pv_name = pv.metadata.name
volume_name = pv.spec.csi.volume_handle # NOQA
volume = client.by_id_volume(volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
wait_delete_pv(core_api, pv_name)
wait_delete_pvc(core_api, pvc['metadata']['name'])
def test_engine_image_incompatible(client, volume_name): # NOQA
images = client.list_engine_image()
assert len(images) == 1
assert images[0]["default"]
assert images[0]["state"] == "ready"
cli_v = images[0]["cliAPIVersion"]
# cli_minv = images[0]["cliAPIMinVersion"]
ctl_v = images[0]["controllerAPIVersion"]
ctl_minv = images[0]["controllerAPIMinVersion"]
data_v = images[0]["dataFormatVersion"]
data_minv = images[0]["dataFormatMinVersion"]
fail_cli_v_image = common.get_compatibility_test_image(
cli_v - 1, cli_v - 1,
ctl_v, ctl_minv,
data_v, data_minv)
img = client.create_engine_image(image=fail_cli_v_image)
img_name = img["name"]
img = wait_for_engine_image_state(client, img_name, "incompatible")
assert img["state"] == "incompatible"
assert img["cliAPIVersion"] == cli_v - 1
assert img["cliAPIMinVersion"] == cli_v - 1
client.delete(img)
fail_cli_minv_image = common.get_compatibility_test_image(
cli_v + 1, cli_v + 1,
ctl_v, ctl_minv,
data_v, data_minv)
img = client.create_engine_image(image=fail_cli_minv_image)
img_name = img["name"]
img = wait_for_engine_image_state(client, img_name, "incompatible")
assert img["state"] == "incompatible"
assert img["cliAPIVersion"] == cli_v + 1
assert img["cliAPIMinVersion"] == cli_v + 1
client.delete(img)
def backing_image_basic_operation_test(client, volume_name, bi_name,
bi_url): # NOQA
"""
Test Backing Image APIs.
1. Create a backing image.
2. Create and attach a Volume with the backing image set.
3. Verify that the all disk states in the backing image are "downloaded".
4. Try to use the API to manually clean up one disk for the backing image
but get failed.
5. Try to use the API to directly delete the backing image
but get failed.
6. Delete the volume.
7. Use the API to manually clean up one disk for the backing image
8. Delete the backing image.
"""
volume = create_and_check_volume(client, volume_name, 3,
str(BACKING_IMAGE_EXT4_SIZE), bi_name)
lht_host_id = get_self_host_id()
volume.attach(hostId=lht_host_id)
volume = wait_for_volume_healthy(client, volume_name)
assert volume.backingImage == bi_name
assert volume.size == str(BACKING_IMAGE_EXT4_SIZE)
random_disk_id = ""
backing_image = client.by_id_backing_image(bi_name)
assert backing_image.sourceType == BACKING_IMAGE_SOURCE_TYPE_DOWNLOAD
assert backing_image.parameters["url"] == bi_url
assert backing_image.currentChecksum != ""
assert not backing_image.deletionTimestamp
assert len(backing_image.diskFileStatusMap) == 3
for disk_id, status in iter(backing_image.diskFileStatusMap.items()):
assert status.state == "ready"
random_disk_id = disk_id
assert random_disk_id != ''
with pytest.raises(Exception):
backing_image.backingImageCleanup(disks=[random_disk_id])
with pytest.raises(Exception):
client.delete(backing_image)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
backing_image = client.by_id_backing_image(bi_name)
backing_image.backingImageCleanup(disks=[random_disk_id])
backing_image = wait_for_backing_image_disk_cleanup(
client, bi_name, random_disk_id)
client.delete(backing_image)
def test_setting_backing_image_auto_cleanup(client, core_api,
volume_name): # NOQA
"""
Test that the Backing Image Cleanup Wait Interval setting works correctly.
The default value of setting `BackingImageCleanupWaitInterval` is 60.
1. Create a backing image.
2. Create multiple volumes using the backing image.
3. Attach all volumes, Then:
1. Wait for all volumes can become running.
2. Verify the correct in all volumes.
3. Verify the backing image disk status map.
4. Verify the only backing image file in each disk is reused by
multiple replicas. The backing image file path is
`<Data path>/<The backing image name>/backing`
4. Unschedule test node to guarantee when replica removed from test node,
no new replica can be rebuilt on the test node.
5. Remove all replicas in one disk.
Wait for 50 seconds.
Then verify nothing changes in the backing image disk state map
(before the cleanup wait interval is passed).
6. Modify `BackingImageCleanupWaitInterval` to a small value. Then verify:
1. The download state of the disk containing no replica becomes
terminating first, and the entry will be removed from the map later.
2. The related backing image file is removed.
3. The download state of other disks keep unchanged.
All volumes still work fine.
7. Delete all volumes. Verify that there will only remain 1 entry in the
backing image disk map
8. Delete the backing image.
"""
# Step 1
create_backing_image_with_matching_url(client, BACKING_IMAGE_NAME,
BACKING_IMAGE_QCOW2_URL)
# Step 2
volume_names = [volume_name + "-1", volume_name + "-2", volume_name + "-3"]
for volume_name in volume_names:
volume = create_and_check_volume(client, volume_name, 3,
str(BACKING_IMAGE_EXT4_SIZE),
BACKING_IMAGE_NAME)
# Step 3
lht_host_id = get_self_host_id()
for volume_name in volume_names:
volume = client.by_id_volume(volume_name)
volume.attach(hostId=lht_host_id)
wait_for_volume_healthy(client, volume_name)
assert volume.backingImage == BACKING_IMAGE_NAME
backing_image = client.by_id_backing_image(BACKING_IMAGE_NAME)
assert len(backing_image.diskFileStatusMap) == 3
for disk_id, status in iter(backing_image.diskFileStatusMap.items()):
assert status.state == "ready"
backing_images_in_disk = os.listdir("/var/lib/longhorn/backing-images")
assert len(backing_images_in_disk) == 1
assert os.path.exists("/var/lib/longhorn/backing-images/{}/backing".format(
backing_images_in_disk[0]))
assert os.path.exists(
"/var/lib/longhorn/backing-images/{}/backing.cfg".format(
backing_images_in_disk[0]))
# Step 4
current_host = client.by_id_node(id=lht_host_id)
client.update(current_host, allowScheduling=False)
wait_for_node_update(client, lht_host_id, "allowScheduling", False)
# Step 5
for volume_name in volume_names:
volume = client.by_id_volume(volume_name)
for replica in volume.replicas:
if replica.hostId == lht_host_id:
replica_name = replica.name
volume.replicaRemove(name=replica_name)
# This wait interval should be smaller than the setting value.
# Otherwise, the backing image files may be cleaned up.
time.sleep(int(BACKING_IMAGE_CLEANUP_WAIT_INTERVAL))
check_backing_image_disk_map_status(client, BACKING_IMAGE_NAME, 3, "ready")
# Step 6
update_setting(client, "backing-image-cleanup-wait-interval", "1")
check_backing_image_disk_map_status(client, BACKING_IMAGE_NAME, 2, "ready")
backing_images_in_disk = os.listdir("/var/lib/longhorn/backing-images")
assert len(backing_images_in_disk) == 0
# Step 7
for volume_name in volume_names:
volume = client.by_id_volume(volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
check_backing_image_disk_map_status(client, BACKING_IMAGE_NAME, 1, "ready")
def test_ha_simple_recovery(client, volume_name): # NOQA
# get a random client
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2)
volume = common.wait_for_volume_detached(client, volume_name)
assert volume["name"] == volume_name
assert volume["size"] == SIZE
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["created"] != ""
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume["endpoint"] == DEV_PATH + volume_name
assert len(volume["replicas"]) == 2
replica0 = volume["replicas"][0]
assert replica0["name"] != ""
replica1 = volume["replicas"][1]
assert replica1["name"] != ""
data = write_random_data(volume["endpoint"])
volume = volume.replicaRemove(name=replica0["name"])
# wait until we saw a replica starts rebuilding
new_replica_found = False
for i in range(RETRY_COUNTS):
v = client.by_id_volume(volume_name)
for r in v["replicas"]:
if r["name"] != replica0["name"] and \
r["name"] != replica1["name"]:
new_replica_found = True
break
if new_replica_found:
break
time.sleep(RETRY_ITERVAL)
assert new_replica_found
volume = common.wait_for_volume_healthy(client, volume_name)
volume = client.by_id_volume(volume_name)
assert volume["state"] == common.VOLUME_STATE_ATTACHED
assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY
assert len(volume["replicas"]) >= 2
found = False
for replica in volume["replicas"]:
if replica["name"] == replica1["name"]:
found = True
break
assert found
check_data(volume['endpoint'], data)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def cleanup_volume(client, vol_name): # NOQA
volume = client.by_id_volume(vol_name)
volume.detach()
client.delete(volume)
common.wait_for_volume_delete(client, vol_name)
def post(self, request):
"""
Manage the POST request, the Master Node will receive a block and it
will check wether it is accepted or not and will add it to the
rest of the blockchain accordingly only
if the one requesting it is a Relay Node (see user and password above).
If the block is rejected because of bad transactions, those transactions
are returned to the relay node that made the request.
If the block is accepted, the new block is sent to all relay nodes.
"""
try:
# request contains the block, and the address of the miner
logger.debug("Block received from %s" % request.data['miner_address'])
block_data = request.data['block']
block = Block.parse(block_data)
except KeyError:
logger.debug("No block given.")
return Response({"errors": "No block given."},
status=status.HTTP_406_NOT_ACCEPTABLE)
except ParseException as e:
logger.debug("Parsing block error.")
return Response({"errors": "%s" % e},
status=status.HTTP_406_NOT_ACCEPTABLE)
(hash_verify, bad_transactions) = self.server.update_blockchain(block)
if hash_verify and len(bad_transactions) == 0: # block is valid
logger.debug("Block '%s' successfully added" % block.header)
data = {'transactions': []}
for transaction in block.transactions:
data['transactions'].append(Transaction.serialize(transaction))
for relay_ip in settings.RELAY_IP:
logger.debug("Sending block '%s' to relay %s" % (block.header, relay_ip))
client.post(relay_ip, 'blockchain', block_data)
client.delete(relay_ip, 'transactions', data)
if self.server.balance >= settings.REWARD:
self.server.balance -= settings.REWARD
miner_transaction = self.server.wallet.create_transaction(
request.data['miner_address'],
settings.REWARD)
client.post(relay_ip, 'transactions', Transaction.serialize(miner_transaction))
response = Response({"detail": "Block successfully added!"},
status=status.HTTP_201_CREATED)
else:
logger.debug("Block '%s' can NOT be added (bad header or bad TXs)." % block.header)
data = {'transactions': []}
if len(bad_transactions) > 0:
for transaction in bad_transactions:
logger.debug("Bad TX '%s'" % transaction.hash)
data['transactions'].append(Transaction.serialize(transaction))
# Send to all relays bad TXs, since the miner can request
# transactions from any relay
for relay_ip in settings.RELAY_IP:
logger.debug("Sending bad TXs to relay %s" % relay_ip)
client.delete(relay_ip, 'transactions', data)
response = Response({"errors": "Bad transactions where found in new block.",
"data": data},
status=status.HTTP_406_NOT_ACCEPTABLE)
else:
response = Response({"errors": "Bad header.",
"data": block_data},
status=status.HTTP_406_NOT_ACCEPTABLE)
return response
def test_replica_zone_anti_affinity(client, core_api, volume_name,
k8s_node_zone_tags): # NOQA
"""
Test replica scheduler with zone anti-affinity
1. Set zone anti-affinity to hard.
2. Label nodes 1 & 2 with same zone label "zone1".
Label node 3 with zone label "zone2".
3. Create a volume with 3 replicas.
4. Wait for volume condition `scheduled` to be false.
5. Label node 2 with zone label "zone3".
6. Wait for volume condition `scheduled` to be success.
7. Clear the volume.
8. Set zone anti-affinity to soft.
9. Change the zone labels on node 1 & 2 & 3 to "zone1".
10. Create a volume.
11. Wait for volume condition `scheduled` to be success.
12. Clean up the replica count, the zone labels and the volume.
"""
wait_longhorn_node_zone_updated(client)
replica_node_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_NODE_SOFT_ANTI_AFFINITY)
client.update(replica_node_soft_anti_affinity_setting, value="false")
replica_zone_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_ZONE_SOFT_ANTI_AFFINITY)
client.update(replica_zone_soft_anti_affinity_setting, value="false")
volume = create_and_check_volume(client, volume_name)
assert volume.conditions.scheduled.status == "False"
assert volume.conditions.scheduled.reason == "ReplicaSchedulingFailure"
lh_nodes = client.list_node()
count = 0
for node in lh_nodes:
count += 1
set_k8s_node_zone_label(core_api, node.name, "lh-zone" + str(count))
wait_longhorn_node_zone_updated(client)
wait_for_volume_condition_scheduled(client, volume_name, "status",
CONDITION_STATUS_TRUE)
replica_zone_soft_anti_affinity_setting = \
client.by_id_setting(SETTING_REPLICA_ZONE_SOFT_ANTI_AFFINITY)
client.update(replica_zone_soft_anti_affinity_setting, value="true")
volume = client.by_id_volume(volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
for node in lh_nodes:
set_k8s_node_zone_label(core_api, node.name, "lh-zone1")
wait_longhorn_node_zone_updated(client)
volume = create_and_check_volume(client, volume_name)
wait_for_volume_condition_scheduled(client, volume_name, "status",
CONDITION_STATUS_TRUE)
def engine_live_upgrade_rollback_test(client,
volume_name,
base_image=""): # NOQA
default_img = common.get_default_engine_image(client)
default_img_name = default_img["name"]
default_img = wait_for_engine_image_ref_count(client, default_img_name, 0)
cli_v = default_img["cliAPIVersion"]
cli_minv = default_img["cliAPIMinVersion"]
ctl_v = default_img["controllerAPIVersion"]
ctl_minv = default_img["controllerAPIMinVersion"]
data_v = default_img["dataFormatVersion"]
data_minv = default_img["dataFormatMinVersion"]
wrong_engine_upgrade_image = common.get_compatibility_test_image(
cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv)
new_img = client.create_engine_image(image=wrong_engine_upgrade_image)
new_img_name = new_img["name"]
new_img = wait_for_engine_image_state(client, new_img_name, "ready")
assert new_img["refCount"] == 0
assert new_img["noRefSince"] != ""
default_img = common.get_default_engine_image(client)
default_img_name = default_img["name"]
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
default_img = wait_for_engine_image_ref_count(client, default_img_name, 1)
assert volume["baseImage"] == base_image
original_engine_image = volume["engineImage"]
assert original_engine_image != wrong_engine_upgrade_image
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
data = write_volume_random_data(volume)
volume.engineUpgrade(image=wrong_engine_upgrade_image)
volume = client.by_id_volume(volume["name"])
assert volume["engineImage"] == wrong_engine_upgrade_image
assert volume["currentImage"] == original_engine_image
with pytest.raises(Exception):
# this will timeout
wait_for_volume_current_image(client, volume_name,
wrong_engine_upgrade_image)
# rollback
volume.engineUpgrade(image=original_engine_image)
volume = wait_for_volume_current_image(client, volume_name,
original_engine_image)
assert volume["engineImage"] == original_engine_image
assert volume["currentImage"] == original_engine_image
engine = get_volume_engine(volume)
assert engine["engineImage"] == original_engine_image
assert engine["currentImage"] == original_engine_image
volume = common.wait_for_volume_replica_count(client, volume_name,
REPLICA_COUNT)
check_volume_data(volume, data)
assert volume["state"] == common.VOLUME_STATE_ATTACHED
assert volume["robustness"] == common.VOLUME_ROBUSTNESS_HEALTHY
# try again, this time let's try detach
volume.engineUpgrade(image=wrong_engine_upgrade_image)
volume = client.by_id_volume(volume["name"])
assert volume["engineImage"] == wrong_engine_upgrade_image
assert volume["currentImage"] == original_engine_image
with pytest.raises(Exception):
# this will timeout
wait_for_volume_current_image(client, volume_name,
wrong_engine_upgrade_image)
volume = volume.detach()
volume = wait_for_volume_current_image(client, volume_name,
wrong_engine_upgrade_image)
# all the images would be updated
assert volume["engineImage"] == wrong_engine_upgrade_image
engine = get_volume_engine(volume)
assert engine["engineImage"] == wrong_engine_upgrade_image
volume = common.wait_for_volume_replica_count(client, volume_name,
REPLICA_COUNT)
for replica in volume["replicas"]:
assert replica["engineImage"] == wrong_engine_upgrade_image
# upgrade to the correct image when offline
volume.engineUpgrade(image=original_engine_image)
volume = client.by_id_volume(volume["name"])
assert volume["engineImage"] == original_engine_image
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert volume["engineImage"] == original_engine_image
assert volume["currentImage"] == original_engine_image
engine = get_volume_engine(volume)
assert engine["engineImage"] == original_engine_image
assert engine["currentImage"] == original_engine_image
for replica in volume["replicas"]:
assert replica["engineImage"] == original_engine_image
assert replica["currentImage"] == original_engine_image
check_volume_data(volume, data)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
client.delete(new_img)
def test_backup_kubernetes_status(set_random_backupstore, client, core_api,
pod): # NOQA
"""
Test that Backups have KubernetesStatus stored properly when there is an
associated PersistentVolumeClaim and Pod.
1. Setup a random backupstore
2. Set settings Longhorn Static StorageClass to `longhorn-static-test`
3. Create a volume and PV/PVC. Verify the StorageClass of PVC
4. Create a Pod using the PVC.
5. Check volume's Kubernetes status to reflect PV/PVC/Pod correctly.
6. Create a backup for the volume.
7. Verify the labels of created backup reflect PV/PVC/Pod status.
8. Restore the backup to a volume. Wait for restoration to complete.
9. Check the volume's Kubernetes Status
1. Make sure the `lastPodRefAt` and `lastPVCRefAt` is snapshot created
time
10. Delete the backup and restored volume.
11. Delete PV/PVC/Pod.
12. Verify volume's Kubernetes Status updated to reflect history data.
13. Attach the volume and create another backup. Verify the labels
14. Verify the volume's Kubernetes status.
15. Restore the previous backup to a new volume. Wait for restoration.
16. Verify the restored volume's Kubernetes status.
1. Make sure `lastPodRefAt` and `lastPVCRefAt` matched volume on step
12
"""
host_id = get_self_host_id()
static_sc_name = "longhorn-static-test"
setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC)
setting = client.update(setting, value=static_sc_name)
assert setting.value == static_sc_name
volume_name = "test-backup-kubernetes-status-pod" # NOQA
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2)
volume = wait_for_volume_detached(client, volume_name)
pod_name = "pod-" + volume_name
pv_name = "pv-" + volume_name
pvc_name = "pvc-" + volume_name
create_pv_for_volume(client, core_api, volume, pv_name)
create_pvc_for_volume(client, core_api, volume, pvc_name)
ret = core_api.list_namespaced_persistent_volume_claim(namespace='default')
pvc_found = False
for item in ret.items:
if item.metadata.name == pvc_name:
pvc_found = item
break
assert pvc_found
assert pvc_found.spec.storage_class_name == static_sc_name
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pvc_name,
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'lastPodRefAt':
'',
'lastPVCRefAt':
'',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
pv_name,
'pvStatus':
'Bound',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_healthy(client, volume_name)
# Create Backup manually instead of calling create_backup since Kubernetes
# is not guaranteed to mount our Volume to the test host.
snap = create_snapshot(client, volume_name)
volume.snapshotBackup(name=snap.name)
wait_for_backup_completion(client, volume_name, snap.name)
_, b = find_backup(client, volume_name, snap.name)
# Check backup label
status = loads(b.labels.get(KUBERNETES_STATUS_LABEL))
assert status == ks
# Check backup volume label
for _ in range(RETRY_COUNTS):
bv = client.by_id_backupVolume(volume_name)
if bv is not None and bv.labels is not None:
break
time.sleep(RETRY_INTERVAL)
assert bv is not None and bv.labels is not None
status = loads(bv.labels.get(KUBERNETES_STATUS_LABEL))
assert status == ks
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b.url)
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
snapshot_created = b.snapshotCreated
ks = {
'lastPodRefAt':
b.snapshotCreated,
'lastPVCRefAt':
b.snapshotCreated,
'namespace':
'default',
'pvcName':
pvc_name,
# Restoration should not apply PersistentVolume data.
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
# We need to compare LastPodRefAt and LastPVCRefAt manually since
# wait_volume_kubernetes_status only checks for empty or non-empty state.
assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"]
assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"]
delete_backup(client, bv.name, b.name)
client.delete(restore)
wait_for_volume_delete(client, restore_name)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, pvc_name)
delete_and_wait_pv(core_api, pv_name)
# With the Pod, PVC, and PV deleted, the Volume should have both Ref
# fields set. Check that a new Backup and Restore will use this instead of
# manually populating the Ref fields.
ks = {
'lastPodRefAt':
'NOT NULL',
'lastPVCRefAt':
'NOT NULL',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
snap = create_snapshot(client, volume_name)
volume.snapshotBackup(name=snap.name)
volume = wait_for_backup_completion(client, volume_name, snap.name)
bv, b = find_backup(client, volume_name, snap.name)
new_b = bv.backupGet(name=b.name)
status = loads(new_b.labels.get(KUBERNETES_STATUS_LABEL))
# Check each field manually, we have no idea what the LastPodRefAt or the
# LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated.
assert status['lastPodRefAt'] != snapshot_created
assert status['lastPVCRefAt'] != snapshot_created
assert status['namespace'] == "default"
assert status['pvcName'] == pvc_name
assert status['pvName'] == ""
assert status['pvStatus'] == ""
assert status['workloadsStatus'] == [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b.url)
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
ks = {
'lastPodRefAt':
status['lastPodRefAt'],
'lastPVCRefAt':
status['lastPVCRefAt'],
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"]
assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"]
# cleanup
backupstore_cleanup(client)
client.delete(restore)
cleanup_volume(client, volume)
def test_recurring_job(clients, volume_name): # NOQA
"""
Test recurring job
1. Setup a random backupstore
2. Create a volume.
3. Create two jobs
1 job 1: snapshot every one minute, retain 2
1 job 2: backup every two minutes, retain 1
4. Attach the volume.
5. Sleep for 5 minutes
6. Verify we have 4 snapshots total
1. 2 snapshots, 1 backup, 1 volume-head
7. Update jobs to replace the backup job
1. New backup job run every one minute, retain 2
8. Sleep for 5 minutes.
9. We should have 6 snapshots
1. 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1
volume-head
10. Make sure we have no more than 5 backups.
1. old backup job may have at most 1 backups
2. new backup job may have at most 3 backups
3. FIXME: Seems we should have at most 4 backups?
11. Make sure we have no more than 2 backups in progress
1. FIXME: Seems we should have at most 1 from the new job?
"""
for host_id, client in iter(clients.items()): # NOQA
break
set_random_backupstore(client)
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2)
volume = common.wait_for_volume_detached(client, volume_name)
jobs = create_jobs1()
volume.recurringUpdate(jobs=jobs)
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
# 5 minutes
time.sleep(300)
check_jobs1_result(volume)
job_backup2 = {
"name": "backup2",
"cron": "* * * * *",
"task": "backup",
"retain": 2
}
volume.recurringUpdate(jobs=[jobs[0], job_backup2])
# 5 minutes
time.sleep(300)
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot.removed is False:
count += 1
# 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head
assert count == 6
complete_backup_number = 0
in_progress_backup_number = 0
volume = client.by_id_volume(volume_name)
for b in volume.backupStatus:
assert b.error == ""
if b.state == "complete":
complete_backup_number += 1
elif b.state == "in_progress":
in_progress_backup_number += 1
assert complete_backup_number <= MAX_BACKUP_STATUS_SIZE
# 1 from job_backup, 1 from job_backup2
assert in_progress_backup_number <= 2
volume = volume.detach()
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def test_engine_image(client, core_api, volume_name): # NOQA
"""
Test Engine Image deployment
1. List Engine Images and validate basic properities.
2. Try deleting default engine image and it should fail.
3. Try creating a duplicate engine image as default and it should fail
4. Get upgrade test image for the same versions
5. Test if the upgrade test image can be deployed and deleted correctly
"""
# can be leftover
default_img = common.get_default_engine_image(client)
default_img_name = default_img.name
default_img = wait_for_engine_image_ref_count(client, default_img_name, 0)
ei_state = get_engine_image_status_value(client, default_img_name)
images = client.list_engine_image()
assert len(images) == 1
assert images[0].default
assert images[0].state == ei_state
assert images[0].refCount == 0
assert images[0].gitCommit != ""
assert images[0].buildDate != ""
cli_v = default_img.cliAPIVersion
cli_minv = default_img.cliAPIMinVersion
ctl_v = default_img.controllerAPIVersion
ctl_minv = default_img.controllerAPIMinVersion
data_v = default_img.dataFormatVersion
data_minv = default_img.dataFormatMinVersion
assert cli_v != 0
assert cli_minv != 0
assert ctl_v != 0
assert ctl_minv != 0
assert data_v != 0
assert data_minv != 0
# delete default image is not allowed
with pytest.raises(Exception) as e:
client.delete(images[0])
assert "the default engine image" in str(e.value)
# duplicate images
with pytest.raises(Exception) as e:
client.create_engine_image(image=default_img.image)
engine_upgrade_image = common.get_upgrade_test_image(
cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv)
# test if engine image can be created and cleaned up successfully
for _ in range(ENGINE_IMAGE_TEST_REPEAT_COUNT):
new_img = client.create_engine_image(image=engine_upgrade_image)
new_img_name = new_img.name
new_img = wait_for_engine_image_state(client, new_img_name, ei_state)
assert not new_img.default
assert new_img.state == ei_state
assert new_img.refCount == 0
assert new_img.cliAPIVersion != 0
assert new_img.cliAPIMinVersion != 0
assert new_img.controllerAPIVersion != 0
assert new_img.controllerAPIMinVersion != 0
assert new_img.dataFormatVersion != 0
assert new_img.dataFormatMinVersion != 0
assert new_img.gitCommit != ""
assert new_img.buildDate != ""
client.delete(new_img)
wait_for_engine_image_deletion(client, core_api, new_img.name)
def ha_backup_deletion_recovery_test(client,
volume_name,
size,
base_image=""): # NOQA
volume = client.create_volume(name=volume_name,
size=size,
numberOfReplicas=2,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
data = write_volume_random_data(volume)
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
_, b = common.find_backup(client, volume_name, snap2["name"])
res_name = common.generate_volume_name()
res_volume = client.create_volume(name=res_name,
size=size,
numberOfReplicas=2,
fromBackup=b["url"])
res_volume = common.wait_for_volume_restoration_completed(
client, res_name)
res_volume = common.wait_for_volume_detached(client, res_name)
res_volume = res_volume.attach(hostId=host_id)
res_volume = common.wait_for_volume_healthy(client, res_name)
check_volume_data(res_volume, data)
snapshots = res_volume.snapshotList()
# only the backup snapshot + volume-head
assert len(snapshots) == 2
backup_snapshot = ""
for snap in snapshots:
if snap["name"] != "volume-head":
backup_snapshot = snap["name"]
assert backup_snapshot != ""
res_volume.snapshotCreate()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 3
res_volume.snapshotDelete(name=backup_snapshot)
res_volume.snapshotPurge()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 2
ha_rebuild_replica_test(client, res_name)
res_volume = res_volume.detach()
res_volume = common.wait_for_volume_detached(client, res_name)
client.delete(res_volume)
common.wait_for_volume_delete(client, res_name)
cleanup_volume(client, volume)
def engine_live_upgrade_rollback_test(client,
core_api,
volume_name,
backing_image=""): # NOQA
default_img = common.get_default_engine_image(client)
default_img_name = default_img.name
default_img = wait_for_engine_image_ref_count(client, default_img_name, 0)
cli_v = default_img.cliAPIVersion
cli_minv = default_img.cliAPIMinVersion
ctl_v = default_img.controllerAPIVersion
ctl_minv = default_img.controllerAPIMinVersion
data_v = default_img.dataFormatVersion
data_minv = default_img.dataFormatMinVersion
wrong_engine_upgrade_image = common.get_compatibility_test_image(
cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv)
new_img = client.create_engine_image(image=wrong_engine_upgrade_image)
new_img_name = new_img.name
ei_status_value = get_engine_image_status_value(client, new_img_name)
new_img = wait_for_engine_image_state(client, new_img_name,
ei_status_value)
assert new_img.refCount == 0
assert new_img.noRefSince != ""
default_img = common.get_default_engine_image(client)
default_img_name = default_img.name
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
backingImage=backing_image)
volume = common.wait_for_volume_detached(client, volume_name)
wait_for_engine_image_ref_count(client, default_img_name, 1)
assert volume.backingImage == backing_image
original_engine_image = volume.engineImage
assert original_engine_image != wrong_engine_upgrade_image
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
data = write_volume_random_data(volume)
volume.engineUpgrade(image=wrong_engine_upgrade_image)
volume = client.by_id_volume(volume.name)
assert volume.engineImage == wrong_engine_upgrade_image
assert volume.currentImage == original_engine_image
with pytest.raises(Exception):
# this will timeout
wait_for_volume_current_image(client, volume_name,
wrong_engine_upgrade_image)
# rollback
volume.engineUpgrade(image=original_engine_image)
wait_for_volume_current_image(client, volume_name, original_engine_image)
volume = wait_for_volume_replicas_mode(client, volume_name, "RW")
assert volume.engineImage == original_engine_image
assert volume.currentImage == original_engine_image
engine = get_volume_engine(volume)
assert engine.engineImage == original_engine_image
assert engine.currentImage == original_engine_image
volume = common.wait_for_volume_replica_count(client, volume_name,
REPLICA_COUNT)
check_volume_data(volume, data)
assert volume.state == common.VOLUME_STATE_ATTACHED
assert volume.robustness == common.VOLUME_ROBUSTNESS_HEALTHY
# try again, this time let's try detach
volume.engineUpgrade(image=wrong_engine_upgrade_image)
volume = client.by_id_volume(volume.name)
assert volume.engineImage == wrong_engine_upgrade_image
assert volume.currentImage == original_engine_image
with pytest.raises(Exception):
# this will timeout
wait_for_volume_current_image(client, volume_name,
wrong_engine_upgrade_image)
volume.detach(hostId="")
volume = wait_for_volume_current_image(client, volume_name,
wrong_engine_upgrade_image)
# all the images would be updated
assert volume.engineImage == wrong_engine_upgrade_image
engine = get_volume_engine(volume)
assert engine.engineImage == wrong_engine_upgrade_image
volume = common.wait_for_volume_replica_count(client, volume_name,
REPLICA_COUNT)
for replica in volume.replicas:
assert replica.engineImage == wrong_engine_upgrade_image
# upgrade to the correct image when offline
volume.engineUpgrade(image=original_engine_image)
volume = wait_for_volume_current_image(client, volume_name,
original_engine_image)
volume = client.by_id_volume(volume.name)
assert volume.engineImage == original_engine_image
volume.attach(hostId=host_id)
common.wait_for_volume_healthy(client, volume_name)
volume = wait_for_volume_replicas_mode(client, volume_name, "RW")
assert volume.engineImage == original_engine_image
assert volume.currentImage == original_engine_image
engine = get_volume_engine(volume)
assert engine.engineImage == original_engine_image
assert engine.currentImage == original_engine_image
check_volume_endpoint(volume)
for replica in volume.replicas:
assert replica.engineImage == original_engine_image
assert replica.currentImage == original_engine_image
check_volume_data(volume, data)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
client.delete(new_img)
wait_for_engine_image_deletion(client, core_api, new_img.name)
def delete_orphan(client, orphan_name): # NOQA
orphan = client.by_id_orphan(orphan_name)
client.delete(orphan)
wait_for_orphan_delete(client, orphan_name)
def engine_live_upgrade_test(client,
core_api,
volume_name,
backing_image=""): # NOQA
default_img = common.get_default_engine_image(client)
default_img_name = default_img.name
default_img = wait_for_engine_image_ref_count(client, default_img_name, 0)
cli_v = default_img.cliAPIVersion
cli_minv = default_img.cliAPIMinVersion
ctl_v = default_img.controllerAPIVersion
ctl_minv = default_img.controllerAPIMinVersion
data_v = default_img.dataFormatVersion
data_minv = default_img.dataFormatMinVersion
engine_upgrade_image = common.get_upgrade_test_image(
cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv)
new_img = client.create_engine_image(image=engine_upgrade_image)
new_img_name = new_img.name
ei_status_value = get_engine_image_status_value(client, new_img_name)
new_img = wait_for_engine_image_state(client, new_img_name,
ei_status_value)
assert new_img.refCount == 0
assert new_img.noRefSince != ""
default_img = common.get_default_engine_image(client)
default_img_name = default_img.name
client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
backingImage=backing_image)
volume = common.wait_for_volume_detached(client, volume_name)
wait_for_engine_image_ref_count(client, default_img_name, 1)
assert volume.name == volume_name
assert volume.backingImage == backing_image
original_engine_image = volume.engineImage
assert original_engine_image != engine_upgrade_image
host_id = get_self_host_id()
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert volume.engineImage == original_engine_image
assert volume.currentImage == original_engine_image
engine = get_volume_engine(volume)
assert engine.engineImage == original_engine_image
assert engine.currentImage == original_engine_image
for replica in volume.replicas:
assert replica.engineImage == original_engine_image
assert replica.currentImage == original_engine_image
data = write_volume_random_data(volume)
volume.engineUpgrade(image=engine_upgrade_image)
wait_for_volume_current_image(client, volume_name, engine_upgrade_image)
# Need to wait for Longhorn to get and update the mode for new replicas
volume = wait_for_volume_replicas_mode(client, volume_name, "RW")
engine = get_volume_engine(volume)
assert engine.engineImage == engine_upgrade_image
check_volume_endpoint(volume)
wait_for_engine_image_ref_count(client, default_img_name, 0)
wait_for_engine_image_ref_count(client, new_img_name, 1)
count = 0
# old replica may be in deletion process
for replica in volume.replicas:
if replica.currentImage == engine_upgrade_image:
count += 1
assert count == REPLICA_COUNT
check_volume_data(volume, data)
volume.detach(hostId="")
volume = common.wait_for_volume_detached(client, volume_name)
assert len(volume.replicas) == REPLICA_COUNT
assert volume.engineImage == engine_upgrade_image
engine = get_volume_engine(volume)
assert engine.engineImage == engine_upgrade_image
for replica in volume.replicas:
assert replica.engineImage == engine_upgrade_image
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert volume.engineImage == engine_upgrade_image
assert volume.currentImage == engine_upgrade_image
engine = get_volume_engine(volume)
assert engine.engineImage == engine_upgrade_image
assert engine.currentImage == engine_upgrade_image
check_volume_endpoint(volume)
for replica in volume.replicas:
assert replica.engineImage == engine_upgrade_image
assert replica.currentImage == engine_upgrade_image
# Make sure detaching didn't somehow interfere with the data.
check_volume_data(volume, data)
volume.engineUpgrade(image=original_engine_image)
wait_for_volume_current_image(client, volume_name, original_engine_image)
volume = wait_for_volume_replicas_mode(client, volume_name, "RW")
engine = get_volume_engine(volume)
assert engine.engineImage == original_engine_image
assert engine.currentImage == original_engine_image
check_volume_endpoint(volume)
wait_for_engine_image_ref_count(client, default_img_name, 1)
new_img = wait_for_engine_image_ref_count(client, new_img_name, 0)
assert volume.engineImage == original_engine_image
count = 0
# old replica may be in deletion process
for replica in volume.replicas:
if replica.engineImage == original_engine_image:
count += 1
assert count == REPLICA_COUNT
check_volume_data(volume, data)
volume.detach(hostId="")
volume = common.wait_for_volume_detached(client, volume_name)
assert len(volume.replicas) == REPLICA_COUNT
assert volume.engineImage == original_engine_image
engine = get_volume_engine(volume)
assert engine.engineImage == original_engine_image
for replica in volume.replicas:
assert replica.engineImage == original_engine_image
client.delete(volume)
wait_for_volume_delete(client, volume_name)
client.delete(new_img)
wait_for_engine_image_deletion(client, core_api, new_img.name)
def test_backup_kubernetes_status(client, core_api, pod): # NOQA
"""
Test that Backups have KubernetesStatus stored properly when there is an
associated PersistentVolumeClaim and Pod.
"""
host_id = get_self_host_id()
static_sc_name = "longhorn-static-test"
setting = client.by_id_setting(SETTING_DEFAULT_LONGHORN_STATIC_SC)
setting = client.update(setting, value=static_sc_name)
assert setting["value"] == static_sc_name
volume_name = "test-backup-kubernetes-status-pod"
client.create_volume(name=volume_name, size=SIZE, numberOfReplicas=2)
volume = wait_for_volume_detached(client, volume_name)
pod_name = "pod-" + volume_name
pv_name = "pv-" + volume_name
pvc_name = "pvc-" + volume_name
create_pv_for_volume(client, core_api, volume, pv_name)
create_pvc_for_volume(client, core_api, volume, pvc_name)
ret = core_api.list_namespaced_persistent_volume_claim(namespace='default')
pvc_found = False
for item in ret.items:
if item.metadata.name == pvc_name:
pvc_found = item
break
assert pvc_found
assert pvc_found.spec.storage_class_name == static_sc_name
pod['metadata']['name'] = pod_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': pvc_name,
},
}]
create_and_wait_pod(core_api, pod)
ks = {
'lastPodRefAt':
'',
'lastPVCRefAt':
'',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
pv_name,
'pvStatus':
'Bound',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_healthy(client, volume_name)
# Create Backup manually instead of calling create_backup since Kubernetes
# is not guaranteed to mount our Volume to the test host.
snap = volume.snapshotCreate()
volume.snapshotBackup(name=snap["name"])
bv, b = find_backup(client, volume_name, snap["name"])
new_b = bv.backupGet(name=b["name"])
status = loads(new_b["labels"].get(KUBERNETES_STATUS_LABEL))
assert status == ks
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b["url"])
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
snapshot_created = b["snapshotCreated"]
ks = {
'lastPodRefAt':
b["snapshotCreated"],
'lastPVCRefAt':
b["snapshotCreated"],
'namespace':
'default',
'pvcName':
pvc_name,
# Restoration should not apply PersistentVolume data.
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
# We need to compare LastPodRefAt and LastPVCRefAt manually since
# wait_volume_kubernetes_status only checks for empty or non-empty state.
assert restore["kubernetesStatus"]["lastPodRefAt"] == ks["lastPodRefAt"]
assert restore["kubernetesStatus"]["lastPVCRefAt"] == ks["lastPVCRefAt"]
bv.backupDelete(name=b["name"])
client.delete(restore)
wait_for_volume_delete(client, restore_name)
delete_and_wait_pod(core_api, pod_name)
delete_and_wait_pvc(core_api, pvc_name)
delete_and_wait_pv(core_api, pv_name)
# With the Pod, PVC, and PV deleted, the Volume should have both Ref
# fields set. Check that a new Backup and Restore will use this instead of
# manually populating the Ref fields.
ks = {
'lastPodRefAt':
'NOT NULL',
'lastPVCRefAt':
'NOT NULL',
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, volume_name, ks)
volume = wait_for_volume_detached(client, volume_name)
volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
snap = volume.snapshotCreate()
volume.snapshotBackup(name=snap["name"])
bv, b = find_backup(client, volume_name, snap["name"])
new_b = bv.backupGet(name=b["name"])
status = loads(new_b["labels"].get(KUBERNETES_STATUS_LABEL))
# Check each field manually, we have no idea what the LastPodRefAt or the
# LastPVCRefAt will be. We just know it shouldn't be SnapshotCreated.
assert status["lastPodRefAt"] != snapshot_created
assert status["lastPVCRefAt"] != snapshot_created
assert status["namespace"] == "default"
assert status["pvcName"] == pvc_name
assert status["pvName"] == ""
assert status["pvStatus"] == ""
assert status["workloadsStatus"] == [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
restore_name = generate_volume_name()
client.create_volume(name=restore_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=b["url"])
wait_for_volume_restoration_completed(client, restore_name)
wait_for_volume_detached(client, restore_name)
ks = {
'lastPodRefAt':
status["lastPodRefAt"],
'lastPVCRefAt':
status["lastPVCRefAt"],
'namespace':
'default',
'pvcName':
pvc_name,
'pvName':
'',
'pvStatus':
'',
'workloadsStatus': [{
'podName': pod_name,
'podStatus': 'Running',
'workloadName': '',
'workloadType': ''
}]
}
wait_volume_kubernetes_status(client, restore_name, ks)
restore = client.by_id_volume(restore_name)
assert restore["kubernetesStatus"]["lastPodRefAt"] == ks["lastPodRefAt"]
assert restore["kubernetesStatus"]["lastPVCRefAt"] == ks["lastPVCRefAt"]
bv.backupDelete(name=b["name"])
client.delete(restore)
cleanup_volume(client, volume)
def cleanup_volume(client, vol_name): # NOQA
volume = client.by_id_volume(vol_name)
volume.detach()
client.delete(volume)
common.wait_for_volume_delete(client, vol_name)
def engine_live_upgrade_test(client, volume_name, base_image=""): # NOQA
default_img = common.get_default_engine_image(client)
default_img_name = default_img["name"]
default_img = wait_for_engine_image_ref_count(client, default_img_name, 0)
cli_v = default_img["cliAPIVersion"]
cli_minv = default_img["cliAPIMinVersion"]
ctl_v = default_img["controllerAPIVersion"]
ctl_minv = default_img["controllerAPIMinVersion"]
data_v = default_img["dataFormatVersion"]
data_minv = default_img["dataFormatMinVersion"]
engine_upgrade_image = common.get_upgrade_test_image(
cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv)
new_img = client.create_engine_image(image=engine_upgrade_image)
new_img_name = new_img["name"]
new_img = wait_for_engine_image_state(client, new_img_name, "ready")
assert new_img["refCount"] == 0
assert new_img["noRefSince"] != ""
default_img = common.get_default_engine_image(client)
default_img_name = default_img["name"]
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
default_img = wait_for_engine_image_ref_count(client, default_img_name, 1)
assert volume["name"] == volume_name
assert volume["baseImage"] == base_image
original_engine_image = volume["engineImage"]
assert original_engine_image != engine_upgrade_image
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert volume["engineImage"] == original_engine_image
assert volume["currentImage"] == original_engine_image
engine = get_volume_engine(volume)
assert engine["engineImage"] == original_engine_image
assert engine["currentImage"] == original_engine_image
for replica in volume["replicas"]:
assert replica["engineImage"] == original_engine_image
assert replica["currentImage"] == original_engine_image
data = write_volume_random_data(volume)
volume.engineUpgrade(image=engine_upgrade_image)
volume = wait_for_volume_current_image(client, volume_name,
engine_upgrade_image)
engine = get_volume_engine(volume)
assert engine["engineImage"] == engine_upgrade_image
default_img = wait_for_engine_image_ref_count(client, default_img_name, 0)
new_img = wait_for_engine_image_ref_count(client, new_img_name, 1)
count = 0
# old replica may be in deletion process
for replica in volume["replicas"]:
if replica["currentImage"] == engine_upgrade_image:
count += 1
assert count == REPLICA_COUNT
check_volume_data(volume, data)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
assert len(volume["replicas"]) == REPLICA_COUNT
assert volume["engineImage"] == engine_upgrade_image
engine = get_volume_engine(volume)
assert engine["engineImage"] == engine_upgrade_image
for replica in volume["replicas"]:
assert replica["engineImage"] == engine_upgrade_image
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
assert volume["engineImage"] == engine_upgrade_image
assert volume["currentImage"] == engine_upgrade_image
engine = get_volume_engine(volume)
assert engine["engineImage"] == engine_upgrade_image
assert engine["currentImage"] == engine_upgrade_image
for replica in volume["replicas"]:
assert replica["engineImage"] == engine_upgrade_image
assert replica["currentImage"] == engine_upgrade_image
# Make sure detaching didn't somehow interfere with the data.
check_volume_data(volume, data)
volume.engineUpgrade(image=original_engine_image)
volume = wait_for_volume_current_image(client, volume_name,
original_engine_image)
engine = get_volume_engine(volume)
assert engine["engineImage"] == original_engine_image
default_img = wait_for_engine_image_ref_count(client, default_img_name, 1)
new_img = wait_for_engine_image_ref_count(client, new_img_name, 0)
assert volume["engineImage"] == original_engine_image
engine = get_volume_engine(volume)
assert engine["engineImage"] == original_engine_image
count = 0
# old replica may be in deletion process
for replica in volume["replicas"]:
if replica["engineImage"] == original_engine_image:
count += 1
assert count == REPLICA_COUNT
check_volume_data(volume, data)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
assert len(volume["replicas"]) == REPLICA_COUNT
assert volume["engineImage"] == original_engine_image
engine = get_volume_engine(volume)
assert engine["engineImage"] == original_engine_image
for replica in volume["replicas"]:
assert replica["engineImage"] == original_engine_image
client.delete(volume)
wait_for_volume_delete(client, volume_name)
client.delete(new_img)
def test_recurring_job(set_random_backupstore, client, volume_name): # NOQA
"""
Test recurring job
1. Setup a random backupstore
2. Create a volume.
3. Create two jobs
1 job 1: snapshot every one minute, retain 2
1 job 2: backup every two minutes, retain 1
4. Attach the volume.
Wait until the 10th second since the beginning of an even minute
5. Write some data. Sleep 2.5 minutes.
Write some data. Sleep 2.5 minutes
6. Verify we have 4 snapshots total
1. 2 snapshots, 1 backup, 1 volume-head
7. Update jobs to replace the backup job
1. New backup job run every one minute, retain 2
8. Write some data. Sleep 2.5 minutes.
Write some data. Sleep 2.5 minutes
9. We should have 6 snapshots
1. 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1
volume-head
10. Make sure there are exactly 4 completed backups.
1. old backup job completed 2 backups
2. new backup job completed 2 backups
11. Make sure we have no backup in progress
"""
'''
The timeline looks like this:
0 1 2 3 4 5 6 7 8 9 10 (minute)
|W | | W | | |W | | W | | | (write data)
| S | S | | S | S | | (job_snap)
| | B | B | | | | | | (job_backup1)
| | | | | | B | B | | (job_backup2)
'''
host_id = get_self_host_id()
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=2)
volume = common.wait_for_volume_detached(client, volume_name)
jobs = create_jobs1()
volume.recurringUpdate(jobs=jobs)
volume = volume.attach(hostId=host_id)
volume = wait_for_volume_healthy(client, volume_name)
# wait until the beginning of an even minute
wait_until_begin_of_an_even_minute()
# wait until the 20th second of an even minute
# make sure that snapshot job happens before the backup job
time.sleep(20)
write_volume_random_data(volume)
time.sleep(120) # 2 minutes
write_volume_random_data(volume)
time.sleep(120) # 2 minutes
check_jobs1_result(volume)
job_backup2 = {
"name": "backup2",
"cron": "* * * * *",
"task": "backup",
"retain": 2
}
volume.recurringUpdate(jobs=[jobs[0], job_backup2])
write_volume_random_data(volume)
time.sleep(120) # 2 minutes
write_volume_random_data(volume)
time.sleep(120) # 2 minutes
snapshots = volume.snapshotList()
count = 0
for snapshot in snapshots:
if snapshot.removed is False:
count += 1
# 2 from job_snap, 1 from job_backup, 2 from job_backup2, 1 volume-head
assert count == 6
complete_backup_number = 0
in_progress_backup_number = 0
volume = client.by_id_volume(volume_name)
for b in volume.backupStatus:
assert b.error == ""
if b.state == "complete":
complete_backup_number += 1
elif b.state == "in_progress":
in_progress_backup_number += 1
# 2 completed backups from job_backup
# 2 completed backups from job_backup2
assert complete_backup_number == 4
assert in_progress_backup_number == 0
volume = volume.detach(hostId="")
common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
def engine_offline_upgrade_test(client, volume_name, base_image=""): # NOQA
default_img = common.get_default_engine_image(client)
default_img_name = default_img["name"]
default_img = wait_for_engine_image_ref_count(client, default_img_name, 0)
cli_v = default_img["cliAPIVersion"]
cli_minv = default_img["cliAPIMinVersion"]
ctl_v = default_img["controllerAPIVersion"]
ctl_minv = default_img["controllerAPIMinVersion"]
data_v = default_img["dataFormatVersion"]
data_minv = default_img["dataFormatMinVersion"]
engine_upgrade_image = common.get_upgrade_test_image(
cli_v, cli_minv, ctl_v, ctl_minv, data_v, data_minv)
new_img = client.create_engine_image(image=engine_upgrade_image)
new_img_name = new_img["name"]
new_img = wait_for_engine_image_state(client, new_img_name, "ready")
assert new_img["refCount"] == 0
assert new_img["noRefSince"] != ""
default_img = common.get_default_engine_image(client)
default_img_name = default_img["name"]
volume = client.create_volume(name=volume_name,
size=SIZE,
numberOfReplicas=REPLICA_COUNT,
baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
default_img = wait_for_engine_image_ref_count(client, default_img_name, 1)
original_engine_image = default_img["image"]
assert volume["name"] == volume_name
assert volume["engineImage"] == original_engine_image
assert volume["currentImage"] == original_engine_image
assert volume["baseImage"] == base_image
# Before our upgrade, write data to the volume first.
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
data = write_volume_random_data(volume)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
volume.engineUpgrade(image=engine_upgrade_image)
volume = wait_for_volume_current_image(client, volume_name,
engine_upgrade_image)
default_img = wait_for_engine_image_ref_count(client, default_img_name, 0)
new_img = wait_for_engine_image_ref_count(client, new_img_name, 1)
# cannot delete a image in use
with pytest.raises(Exception) as e:
client.delete(new_img)
assert "while being used" in str(e.value)
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
engine = get_volume_engine(volume)
assert engine["engineImage"] == engine_upgrade_image
assert engine["currentImage"] == engine_upgrade_image
for replica in volume["replicas"]:
assert replica["engineImage"] == engine_upgrade_image
assert replica["currentImage"] == engine_upgrade_image
check_volume_data(volume, data)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
volume.engineUpgrade(image=original_engine_image)
volume = wait_for_volume_current_image(client, volume_name,
original_engine_image)
engine = get_volume_engine(volume)
assert volume["engineImage"] == original_engine_image
assert engine["engineImage"] == original_engine_image
for replica in volume["replicas"]:
assert replica["engineImage"] == original_engine_image
default_img = wait_for_engine_image_ref_count(client, default_img_name, 1)
new_img = wait_for_engine_image_ref_count(client, new_img_name, 0)
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
engine = get_volume_engine(volume)
assert engine["engineImage"] == original_engine_image
assert engine["currentImage"] == original_engine_image
for replica in volume["replicas"]:
assert replica["engineImage"] == original_engine_image
assert replica["currentImage"] == original_engine_image
check_volume_data(volume, data)
client.delete(volume)
wait_for_volume_delete(client, volume_name)
client.delete(new_img)
def ha_backup_deletion_recovery_test(client, volume_name, size, base_image=""): # NOQA
volume = client.create_volume(name=volume_name, size=size,
numberOfReplicas=2, baseImage=base_image)
volume = common.wait_for_volume_detached(client, volume_name)
host_id = get_self_host_id()
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
data = write_volume_random_data(volume)
snap2 = volume.snapshotCreate()
volume.snapshotCreate()
volume.snapshotBackup(name=snap2["name"])
_, b = common.find_backup(client, volume_name, snap2["name"])
res_name = common.generate_volume_name()
res_volume = client.create_volume(name=res_name, size=size,
numberOfReplicas=2,
fromBackup=b["url"])
res_volume = common.wait_for_volume_detached(client, res_name)
res_volume = res_volume.attach(hostId=host_id)
res_volume = common.wait_for_volume_healthy(client, res_name)
check_volume_data(res_volume, data)
snapshots = res_volume.snapshotList()
# only the backup snapshot + volume-head
assert len(snapshots) == 2
backup_snapshot = ""
for snap in snapshots:
if snap["name"] != "volume-head":
backup_snapshot = snap["name"]
assert backup_snapshot != ""
res_volume.snapshotCreate()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 3
res_volume.snapshotDelete(name=backup_snapshot)
res_volume.snapshotPurge()
snapshots = res_volume.snapshotList()
assert len(snapshots) == 2
ha_rebuild_replica_test(client, res_name)
res_volume = res_volume.detach()
res_volume = common.wait_for_volume_detached(client, res_name)
client.delete(res_volume)
common.wait_for_volume_delete(client, res_name)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, volume_name)
client.delete(volume)
common.wait_for_volume_delete(client, volume_name)
volumes = client.list_volume()
assert len(volumes) == 0
