def test_empty_backup_volume(clients): # NOQA
for host_id, client in clients.iteritems():
break
lht_hostId = get_self_host_id()
volName = generate_volume_name()
volume = create_and_check_volume(client, volName)
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, volName)
bv, b1, snap1, _ = create_backup(client, volName)
bv.backupDelete(name=b1["name"])
common.wait_for_backup_delete(b1["name"], bv)
backup_list = bv.backupList()
assert len(backup_list) == 0
# test the empty backup volume can recreate backup
_, b2, snap2, _ = create_backup(client, volName)
# test the empty backup volume is still deletable
bv.backupDelete(name=b2["name"])
common.wait_for_backup_delete(b1["name"], bv)
bv = client.by_id_backupVolume(volName)
client.delete(bv)
common.wait_for_backup_volume_delete(client, volName)
cleanup_volume(client, volume)
def backup_test(
dev,
address, # NOQA
volume_name,
engine_name,
backup_target):
offset = 0
length = 128
snap1_data = random_string(length)
verify_data(dev, offset, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
backup1_info = create_backup(address, snap1, backup_target)
assert backup1_info["VolumeName"] == volume_name
assert backup1_info["Size"] == BLOCK_SIZE_STR
snap2_data = random_string(length)
verify_data(dev, offset, snap2_data)
snap2_checksum = checksum_dev(dev)
snap2 = cmd.snapshot_create(address)
backup2_info = create_backup(address, snap2, backup_target)
assert backup2_info["VolumeName"] == volume_name
assert backup2_info["Size"] == BLOCK_SIZE_STR
snap3_data = random_string(length)
verify_data(dev, offset, snap3_data)
snap3_checksum = checksum_dev(dev)
snap3 = cmd.snapshot_create(address)
backup3_info = create_backup(address, snap3, backup_target)
assert backup3_info["VolumeName"] == volume_name
assert backup3_info["Size"] == BLOCK_SIZE_STR
restore_with_frontend(address, engine_name, backup3_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap3_data
c = checksum_dev(dev)
assert c == snap3_checksum
rm_backups(address, engine_name, [backup3_info["URL"]])
restore_with_frontend(address, engine_name, backup1_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = checksum_dev(dev)
assert c == snap1_checksum
rm_backups(address, engine_name, [backup1_info["URL"]])
restore_with_frontend(address, engine_name, backup2_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap2_data
c = checksum_dev(dev)
assert c == snap2_checksum
rm_backups(address, engine_name, [backup2_info["URL"]])
def backup_hole_with_backing_file_test(
backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
offset1 = 512
length1 = 256
offset2 = 640
length2 = 256
boundary_offset = 0
boundary_length = 4100 # just pass 4096 into next 4k
hole_offset = 2 * 1024 * 1024
hole_length = 1024
snap1_data = random_string(length1)
verify_data(dev, offset1, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
boundary_data_backup1 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup1 = read_dev(dev, hole_offset, hole_length)
backup1_info = create_backup(address, snap1, backup_target)
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
snap2_checksum = checksum_dev(dev)
snap2 = cmd.snapshot_create(address)
boundary_data_backup2 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup2 = read_dev(dev, hole_offset, hole_length)
backup2_info = create_backup(address, snap2, backup_target)
restore_with_frontend(address, ENGINE_BACKING_NAME, backup1_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup1
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup1
c = checksum_dev(dev)
assert c == snap1_checksum
restore_with_frontend(address, ENGINE_BACKING_NAME, backup2_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup2
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup2
c = checksum_dev(dev)
assert c == snap2_checksum
def backup_with_backing_file_test(
backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
offset = 0
length = 256
snap0 = cmd.snapshot_create(address)
before = read_dev(dev, offset, length)
assert before != ""
snap0_checksum = checksum_dev(dev)
exists = read_from_backing_file(offset, length)
assert before == exists
backup0_info = create_backup(address, snap0, backup_target)
assert backup0_info["VolumeName"] == VOLUME_BACKING_NAME
backup_test(dev, address, VOLUME_BACKING_NAME, ENGINE_BACKING_NAME,
backup_target)
restore_with_frontend(address, ENGINE_BACKING_NAME, backup0_info["URL"])
after = read_dev(dev, offset, length)
assert before == after
c = checksum_dev(dev)
assert c == snap0_checksum
rm_backups(address, ENGINE_BACKING_NAME, [backup0_info["URL"]])
def test_backup_volume_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
offset = 0
length = 128
address = grpc_controller.address
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
snap_data = random_string(length)
verify_data(dev, offset, snap_data)
snap = cmd.snapshot_create(address)
backup_info = create_backup(address, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME, backup_target)
assert "cannot find" in info[VOLUME_NAME]["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_deleting_backup_volume(clients): # NOQA
for host_id, client in clients.iteritems():
break
lht_hostId = get_self_host_id()
volName = generate_volume_name()
volume = create_and_check_volume(client, volName)
volume.attach(hostId=lht_hostId)
volume = common.wait_for_volume_healthy(client, volName)
bv, _, snap1, _ = create_backup(client, volName)
_, _, snap2, _ = create_backup(client, volName)
bv = client.by_id_backupVolume(volName)
client.delete(bv)
common.wait_for_backup_volume_delete(client, volName)
cleanup_volume(client, volume)
def snapshot_tree_backup_test(
backup_target,
engine_name, # NOQA
grpc_controller,
grpc_replica1,
grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
offset = 0
length = 128
backup = {}
snap, data = snapshot_tree_build(dev, address, engine_name, offset, length)
backup["0b"] = create_backup(address, snap["0b"], backup_target)["URL"]
backup["0c"] = create_backup(address, snap["0c"], backup_target)["URL"]
backup["1c"] = create_backup(address, snap["1c"], backup_target)["URL"]
backup["2b"] = create_backup(address, snap["2b"], backup_target)["URL"]
backup["2c"] = create_backup(address, snap["2c"], backup_target)["URL"]
backup["3c"] = create_backup(address, snap["3c"], backup_target)["URL"]
snapshot_tree_verify_backup_node(dev, address, engine_name, offset, length,
backup, data, "0b")
snapshot_tree_verify_backup_node(dev, address, engine_name, offset, length,
backup, data, "0c")
snapshot_tree_verify_backup_node(dev, address, engine_name, offset, length,
backup, data, "1c")
snapshot_tree_verify_backup_node(dev, address, engine_name, offset, length,
backup, data, "2b")
snapshot_tree_verify_backup_node(dev, address, engine_name, offset, length,
backup, data, "2c")
snapshot_tree_verify_backup_node(dev, address, engine_name, offset, length,
backup, data, "3c")
def backup_labels_test(clients,
random_labels,
volume_name,
size=SIZE,
base_image=""): # NOQA
for _, client in clients.iteritems():
break
host_id = get_self_host_id()
volume = create_and_check_volume(client, volume_name, 2, size, base_image)
volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, volume_name)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
# test backupTarget for multiple settings
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
setting = client.update(setting, value=backupsettings[0])
assert setting["value"] == backupsettings[0]
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value=backupsettings[1])
assert credential["value"] == backupsettings[1]
else:
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
credential = client.by_id_setting(
common.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
credential = client.update(credential, value="")
assert credential["value"] == ""
bv, b, _, _ = create_backup(client, volume_name, labels=random_labels)
# If we're running the test with a BaseImage, check that this Label is
# set properly.
backup = bv.backupGet(name=b["name"])
if base_image:
assert backup["labels"].get(common.BASE_IMAGE_LABEL) == base_image
# One extra Label from the BaseImage being set.
assert len(backup["labels"]) == len(random_labels) + 1
else:
assert len(backup["labels"]) == len(random_labels)
cleanup_volume(client, volume)
def backupstore_test(client, host_id, volname, size):
bv, b, snap2, data = create_backup(client, volname)
# test restore
restoreName = generate_volume_name()
volume = client.create_volume(name=restoreName,
size=size,
numberOfReplicas=2,
fromBackup=b["url"])
volume = common.wait_for_volume_restoration_completed(client, restoreName)
volume = common.wait_for_volume_detached(client, restoreName)
assert volume["name"] == restoreName
assert volume["size"] == size
assert volume["numberOfReplicas"] == 2
assert volume["state"] == "detached"
assert volume["initialRestorationRequired"] is False
volume = volume.attach(hostId=host_id)
volume = common.wait_for_volume_healthy(client, restoreName)
check_volume_data(volume, data)
volume = volume.detach()
volume = common.wait_for_volume_detached(client, restoreName)
bv.backupDelete(name=b["name"])
backups = bv.backupList()
found = False
for b in backups:
if b["snapshotName"] == snap2["name"]:
found = True
break
assert not found
volume = wait_for_volume_status(client, volume["name"], "lastBackup", "")
assert volume["lastBackupAt"] == ""
client.delete(volume)
volume = wait_for_volume_delete(client, restoreName)
def test_listing_backup_volume(clients, base_image=""): # NOQA
for host_id, client in clients.iteritems():
break
lht_hostId = get_self_host_id()
# create 3 volumes.
volume1_name = generate_volume_name()
volume2_name = generate_volume_name()
volume3_name = generate_volume_name()
volume1 = create_and_check_volume(client, volume1_name)
volume2 = create_and_check_volume(client, volume2_name)
volume3 = create_and_check_volume(client, volume3_name)
volume1.attach(hostId=lht_hostId)
volume1 = common.wait_for_volume_healthy(client, volume1_name)
volume2.attach(hostId=lht_hostId)
volume2 = common.wait_for_volume_healthy(client, volume2_name)
volume3.attach(hostId=lht_hostId)
volume3 = common.wait_for_volume_healthy(client, volume3_name)
# we only test NFS here.
# Since it is difficult to directly remove volume.cfg from s3 buckets
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
backupstores = common.get_backupstore_url()
for backupstore in backupstores:
if common.is_backupTarget_nfs(backupstore):
updated = False
for i in range(RETRY_COMMAND_COUNT):
nfs_url = backupstore.strip("nfs://")
setting = client.update(setting, value=backupstore)
assert setting["value"] == backupstore
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
if "nfs" in setting["value"]:
updated = True
break
assert updated
_, _, snap1, _ = create_backup(client, volume1_name)
_, _, snap2, _ = create_backup(client, volume2_name)
_, _, snap3, _ = create_backup(client, volume3_name)
# invalidate backup volume 1 by renaming volume.cfg to volume.cfg.tmp
cmd = ["mkdir", "-p", "/mnt/nfs"]
subprocess.check_output(cmd)
cmd = ["mount", "-t", "nfs4", nfs_url, "/mnt/nfs"]
subprocess.check_output(cmd)
cmd = ["find", "/mnt/nfs", "-type", "d", "-name", volume1_name]
volume1_backup_volume_path = subprocess.check_output(cmd).strip()
cmd = ["find", volume1_backup_volume_path, "-name", "volume.cfg"]
volume1_backup_volume_cfg_path = subprocess.check_output(cmd).strip()
cmd = [
"mv", volume1_backup_volume_cfg_path,
volume1_backup_volume_cfg_path + ".tmp"
]
subprocess.check_output(cmd)
subprocess.check_output(["sync"])
found1 = found2 = found3 = False
for i in range(RETRY_COUNTS):
bvs = client.list_backupVolume()
for bv in bvs:
if bv["name"] == volume1_name:
if "error" in bv.messages:
assert "volume.cfg" in bv.messages["error"].lower()
found1 = True
elif bv["name"] == volume2_name:
assert not bv.messages
found2 = True
elif bv["name"] == volume3_name:
assert not bv.messages
found3 = True
if found1 & found2 & found3:
break
time.sleep(RETRY_INTERVAL)
assert found1 & found2 & found3
cmd = [
"mv", volume1_backup_volume_cfg_path + ".tmp",
volume1_backup_volume_cfg_path
]
subprocess.check_output(cmd)
subprocess.check_output(["sync"])
found = False
for i in range(RETRY_COMMAND_COUNT):
try:
bv1, b1 = common.find_backup(client, volume1_name, snap1["name"])
found = True
break
except Exception:
time.sleep(1)
assert found
bv1.backupDelete(name=b1["name"])
for i in range(RETRY_COMMAND_COUNT):
found = False
backups1 = bv1.backupList()
for b in backups1:
if b["snapshotName"] == snap1["name"]:
found = True
break
assert not found
bv2, b2 = common.find_backup(client, volume2_name, snap2["name"])
bv2.backupDelete(name=b2["name"])
for i in range(RETRY_COMMAND_COUNT):
found = False
backups2 = bv2.backupList()
for b in backups2:
if b["snapshotName"] == snap2["name"]:
found = True
break
assert not found
bv3, b3 = common.find_backup(client, volume3_name, snap3["name"])
bv3.backupDelete(name=b3["name"])
for i in range(RETRY_COMMAND_COUNT):
found = False
backups3 = bv3.backupList()
for b in backups3:
if b["snapshotName"] == snap3["name"]:
found = True
break
assert not found
volume1.detach()
volume1 = common.wait_for_volume_detached(client, volume1_name)
client.delete(volume1)
wait_for_volume_delete(client, volume1_name)
volume2.detach()
volume2 = common.wait_for_volume_detached(client, volume2_name)
client.delete(volume2)
wait_for_volume_delete(client, volume2_name)
volume3.detach()
volume3 = common.wait_for_volume_detached(client, volume3_name)
client.delete(volume3)
wait_for_volume_delete(client, volume3_name)
volumes = client.list_volume()
assert len(volumes) == 0
def restore_inc_test(client, core_api, volume_name, pod): # NOQA
std_volume = create_and_check_volume(client, volume_name, 2, SIZE)
lht_host_id = get_self_host_id()
std_volume.attach(hostId=lht_host_id)
std_volume = common.wait_for_volume_healthy(client, volume_name)
with pytest.raises(Exception) as e:
std_volume.activate(frontend="blockdev")
assert "already in active mode" in str(e.value)
data0 = {'len': 4 * 1024, 'pos': 0}
data0['content'] = common.generate_random_data(data0['len'])
bv, backup0, _, data0 = create_backup(client, volume_name, data0)
sb_volume0_name = "sb-0-" + volume_name
sb_volume1_name = "sb-1-" + volume_name
sb_volume2_name = "sb-2-" + volume_name
client.create_volume(name=sb_volume0_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=backup0['url'],
frontend="",
standby=True)
client.create_volume(name=sb_volume1_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=backup0['url'],
frontend="",
standby=True)
client.create_volume(name=sb_volume2_name,
size=SIZE,
numberOfReplicas=2,
fromBackup=backup0['url'],
frontend="",
standby=True)
common.wait_for_volume_restoration_completed(client, sb_volume0_name)
common.wait_for_volume_restoration_completed(client, sb_volume1_name)
common.wait_for_volume_restoration_completed(client, sb_volume2_name)
sb_volume0 = common.wait_for_volume_healthy(client, sb_volume0_name)
sb_volume1 = common.wait_for_volume_healthy(client, sb_volume1_name)
sb_volume2 = common.wait_for_volume_healthy(client, sb_volume2_name)
for i in range(RETRY_COUNTS):
sb_volume0 = client.by_id_volume(sb_volume0_name)
sb_volume1 = client.by_id_volume(sb_volume1_name)
sb_volume2 = client.by_id_volume(sb_volume2_name)
sb_engine0 = get_volume_engine(sb_volume0)
sb_engine1 = get_volume_engine(sb_volume1)
sb_engine2 = get_volume_engine(sb_volume2)
if sb_volume0["lastBackup"] != backup0["name"] or \
sb_volume1["lastBackup"] != backup0["name"] or \
sb_volume2["lastBackup"] != backup0["name"] or \
sb_engine0["lastRestoredBackup"] != backup0["name"] or \
sb_engine1["lastRestoredBackup"] != backup0["name"] or \
sb_engine2["lastRestoredBackup"] != backup0["name"]:
time.sleep(RETRY_INTERVAL)
else:
break
assert sb_volume0["standby"] is True
assert sb_volume0["lastBackup"] == backup0["name"]
assert sb_volume0["frontend"] == ""
assert sb_volume0["disableFrontend"] is True
assert sb_volume0["initialRestorationRequired"] is False
sb_engine0 = get_volume_engine(sb_volume0)
assert sb_engine0["lastRestoredBackup"] == backup0["name"]
assert sb_engine0["requestedBackupRestore"] == backup0["name"]
assert sb_volume1["standby"] is True
assert sb_volume1["lastBackup"] == backup0["name"]
assert sb_volume1["frontend"] == ""
assert sb_volume1["disableFrontend"] is True
assert sb_volume1["initialRestorationRequired"] is False
sb_engine1 = get_volume_engine(sb_volume1)
assert sb_engine1["lastRestoredBackup"] == backup0["name"]
assert sb_engine1["requestedBackupRestore"] == backup0["name"]
assert sb_volume2["standby"] is True
assert sb_volume2["lastBackup"] == backup0["name"]
assert sb_volume2["frontend"] == ""
assert sb_volume2["disableFrontend"] is True
assert sb_volume2["initialRestorationRequired"] is False
sb_engine2 = get_volume_engine(sb_volume2)
assert sb_engine2["lastRestoredBackup"] == backup0["name"]
assert sb_engine2["requestedBackupRestore"] == backup0["name"]
sb0_snaps = sb_volume0.snapshotList()
assert len(sb0_snaps) == 2
for s in sb0_snaps:
if s['name'] != "volume-head":
sb0_snap = s
assert sb0_snaps
with pytest.raises(Exception) as e:
sb_volume0.snapshotCreate()
assert "cannot create snapshot for standby volume" in str(e.value)
with pytest.raises(Exception) as e:
sb_volume0.snapshotRevert(name=sb0_snap["name"])
assert "cannot revert snapshot for standby volume" in str(e.value)
with pytest.raises(Exception) as e:
sb_volume0.snapshotDelete(name=sb0_snap["name"])
assert "cannot delete snapshot for standby volume" in str(e.value)
with pytest.raises(Exception) as e:
sb_volume0.snapshotBackup(name=sb0_snap["name"])
assert "cannot create backup for standby volume" in str(e.value)
with pytest.raises(Exception) as e:
sb_volume0.pvCreate(pvName=sb_volume0_name)
assert "cannot create PV for standby volume" in str(e.value)
with pytest.raises(Exception) as e:
sb_volume0.pvcCreate(pvcName=sb_volume0_name)
assert "cannot create PVC for standby volume" in str(e.value)
setting = client.by_id_setting(common.SETTING_BACKUP_TARGET)
with pytest.raises(Exception) as e:
client.update(setting, value="random.backup.target")
assert "cannot modify BackupTarget " \
"since there are existing standby volumes" in str(e.value)
with pytest.raises(Exception) as e:
sb_volume0.activate(frontend="wrong_frontend")
assert "invalid frontend" in str(e.value)
activate_standby_volume(client, sb_volume0_name)
sb_volume0 = client.by_id_volume(sb_volume0_name)
sb_volume0.attach(hostId=lht_host_id)
sb_volume0 = common.wait_for_volume_healthy(client, sb_volume0_name)
check_volume_data(sb_volume0, data0, False)
zero_string = b'\x00'.decode('utf-8')
_, backup1, _, data1 = create_backup(client, volume_name, {
'len': 2 * 1024,
'pos': 0,
'content': zero_string * 2 * 1024
})
# use this api to update field `last backup`
client.list_backupVolume()
check_volume_last_backup(client, sb_volume1_name, backup1['name'])
activate_standby_volume(client, sb_volume1_name)
sb_volume1 = client.by_id_volume(sb_volume1_name)
sb_volume1.attach(hostId=lht_host_id)
sb_volume1 = common.wait_for_volume_healthy(client, sb_volume1_name)
data0_modified = {
'len': data0['len'] - data1['len'],
'pos': data1['len'],
'content': data0['content'][data1['len']:],
}
check_volume_data(sb_volume1, data0_modified, False)
check_volume_data(sb_volume1, data1)
data2 = {'len': 1 * 1024 * 1024, 'pos': 0}
data2['content'] = common.generate_random_data(data2['len'])
_, backup2, _, data2 = create_backup(client, volume_name, data2)
client.list_backupVolume()
check_volume_last_backup(client, sb_volume2_name, backup2['name'])
activate_standby_volume(client, sb_volume2_name)
sb_volume2 = client.by_id_volume(sb_volume2_name)
sb_volume2.attach(hostId=lht_host_id)
sb_volume2 = common.wait_for_volume_healthy(client, sb_volume2_name)
check_volume_data(sb_volume2, data2)
# allocated this active volume to a pod
sb_volume2.detach()
sb_volume2 = common.wait_for_volume_detached(client, sb_volume2_name)
create_pv_for_volume(client, core_api, sb_volume2, sb_volume2_name)
create_pvc_for_volume(client, core_api, sb_volume2, sb_volume2_name)
sb_volume2_pod_name = "pod-" + sb_volume2_name
pod['metadata']['name'] = sb_volume2_pod_name
pod['spec']['volumes'] = [{
'name':
pod['spec']['containers'][0]['volumeMounts'][0]['name'],
'persistentVolumeClaim': {
'claimName': sb_volume2_name,
},
}]
create_and_wait_pod(core_api, pod)
sb_volume2 = client.by_id_volume(sb_volume2_name)
k_status = sb_volume2["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert k_status['pvName'] == sb_volume2_name
assert k_status['pvStatus'] == 'Bound'
assert len(workloads) == 1
for i in range(RETRY_COUNTS):
if workloads[0]['podStatus'] == 'Running':
break
time.sleep(RETRY_INTERVAL)
sb_volume2 = client.by_id_volume(sb_volume2_name)
k_status = sb_volume2["kubernetesStatus"]
workloads = k_status['workloadsStatus']
assert len(workloads) == 1
assert workloads[0]['podName'] == sb_volume2_pod_name
assert workloads[0]['podStatus'] == 'Running'
assert not workloads[0]['workloadName']
assert not workloads[0]['workloadType']
assert k_status['namespace'] == 'default'
assert k_status['pvcName'] == sb_volume2_name
assert not k_status['lastPVCRefAt']
assert not k_status['lastPodRefAt']
delete_and_wait_pod(core_api, sb_volume2_pod_name)
delete_and_wait_pvc(core_api, sb_volume2_name)
delete_and_wait_pv(core_api, sb_volume2_name)
# cleanup
std_volume.detach()
sb_volume0.detach()
sb_volume1.detach()
std_volume = common.wait_for_volume_detached(client, volume_name)
sb_volume0 = common.wait_for_volume_detached(client, sb_volume0_name)
sb_volume1 = common.wait_for_volume_detached(client, sb_volume1_name)
sb_volume2 = common.wait_for_volume_detached(client, sb_volume2_name)
bv.backupDelete(name=backup2["name"])
bv.backupDelete(name=backup1["name"])
bv.backupDelete(name=backup0["name"])
client.delete(std_volume)
client.delete(sb_volume0)
client.delete(sb_volume1)
client.delete(sb_volume2)
wait_for_volume_delete(client, volume_name)
wait_for_volume_delete(client, sb_volume0_name)
wait_for_volume_delete(client, sb_volume1_name)
wait_for_volume_delete(client, sb_volume2_name)
volumes = client.list_volume()
assert len(volumes) == 0
def restore_to_file_with_backing_file_test(
backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
backing_dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
length0 = 4 * 1024
length1 = 256
length2 = 128
offset0 = 0
offset1 = length1 + offset0
offset2 = length2 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot.
# data in output image == data in backing
check_backing()
check_empty_volume(backing_dev)
snap0 = cmd.snapshot_create(address)
backup = create_backup(address, snap0, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset0, length0)
dev_checksum = checksum_dev(backing_dev)
assert volume_data != ""
assert volume_data == backing_data
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output0_raw = read_file(output_raw_path, offset0, length0)
output0_checksum = checksum_data(read_file(output_raw_path, 0, SIZE))
assert output0_raw == backing_data
assert output0_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output0_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
output0_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output0_qcow2 == backing_data
assert output0_qcow2 == volume_data
assert output0_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
rm_backups(address, ENGINE_BACKING_NAME, [backup])
# create 1 snapshot with 256B data.
# output = snap1(offset0, length1) + backing(offset1, ...)
snap1_data = random_string(length1)
verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
backup = create_backup(address, snap1, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset1, length0 - offset1)
dev_checksum = checksum_dev(backing_dev)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output1_raw_snap1 = read_file(output_raw_path, offset0, length1)
output1_raw_backing = read_file(output_raw_path, offset1,
length0 - offset1)
output1_checksum = checksum_data(read_file(output_raw_path, 0, SIZE))
assert output1_raw_snap1 == snap1_data
assert output1_raw_backing == backing_data
assert output1_raw_snap1 + output1_raw_backing == volume_data
assert output1_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output1_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output1_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output1_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output1_qcow2_snap1 == snap1_data
assert output1_qcow2_backing == backing_data
assert output1_qcow2_snap1 + output1_qcow2_backing == volume_data
assert output1_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
rm_snaps(address, [snap1])
rm_backups(address, ENGINE_BACKING_NAME, [backup])
check_backing()
check_empty_volume(backing_dev)
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2) + backing(offset2, ...)
snap1_data = random_string(length1)
verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
snap2_data = random_string(length2)
verify_data(backing_dev, offset0, snap2_data)
snap2 = cmd.snapshot_create(address)
backup = create_backup(address, snap2, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset1, length0 - offset1)
dev_checksum = checksum_dev(backing_dev)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(output_raw_path, offset0, length2)
output2_raw_snap1 = read_file(output_raw_path, offset2, length1 - length2)
output2_raw_backing = read_file(output_raw_path, offset1,
length0 - offset1)
output2_checksum = checksum_data(read_file(output_raw_path, 0, SIZE))
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset2:length1]
assert output2_raw_backing == backing_data
assert \
volume_data == \
output2_raw_snap2 + output2_raw_snap1 + output2_raw_backing
assert output2_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length2)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset2, length1 - length2)
output2_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output2_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output2_qcow2_snap2 == snap2_data
assert output2_qcow2_snap1 == snap1_data[offset2:length1]
assert output2_qcow2_backing == backing_data
assert \
volume_data == \
output2_qcow2_snap2 + output2_qcow2_snap1 + output1_qcow2_backing
assert output2_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
rm_snaps(address, [snap1, snap2])
rm_backups(address, ENGINE_BACKING_NAME, [backup])
check_backing()
check_empty_volume(backing_dev)
def restore_to_file_without_backing_file_test(
backup_target, # NOQA
grpc_controller, # NOQA
grpc_replica1, # NOQA
grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
length0 = 256
length1 = 128
offset0 = 0
offset1 = length1 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot for converting to init state.
snap0 = cmd.snapshot_create(address)
# create 1 snapshot with 256B data.
# output = snap2(offset0, length1)
snap1_data = random_string(length0)
verify_data(dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
backup = create_backup(address, snap1, backup_target)["URL"]
cmd.restore_to_file(address, backup, "", output_raw_path, IMAGE_FORMAT_RAW)
output1_raw = read_file(output_raw_path, offset0, length0)
assert output1_raw == snap1_data
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, "", output_qcow2_path,
IMAGE_FORMAT_QCOW2)
output1_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
assert output1_qcow2 == snap1_data
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_NAME, snap0)
rm_snaps(address, [snap1])
rm_backups(address, ENGINE_NAME, [backup])
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2)
snap1_data = random_string(length0)
verify_data(dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
snap2_data = random_string(length1)
verify_data(dev, offset0, snap2_data)
snap2 = cmd.snapshot_create(address)
backup = create_backup(address, snap2, backup_target)["URL"]
cmd.restore_to_file(address, backup, "", output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(output_raw_path, offset0, length1)
output2_raw_snap1 = read_file(output_raw_path, offset1, length0 - length1)
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset1:length0]
cmd.restore_to_file(address, backup, "", output_qcow2_path,
IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - length1)
assert output2_qcow2_snap2 == snap2_data
assert output2_qcow2_snap1 == snap1_data[offset1:length0]
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_NAME, snap0)
rm_snaps(address, [snap1, snap2])
rm_backups(address, ENGINE_NAME, [backup])
def restore_inc_test(
grpc_engine_manager, # NOQA
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2, # NOQA
grpc_dr_controller, # NOQA
grpc_dr_replica1,
grpc_dr_replica2, # NOQA
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(grpc_engine_manager, 0, snap0_data,
grpc_dr_controller)
# mock restore crash/error
delta_file1 = "volume-delta-" + backup0_name + ".img"
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk + ".tmp"]
subprocess.check_output(command).strip()
# should fail
is_failed = False
cmd.restore_inc(dr_address, backup1, backup0_name)
for i in range(RETRY_COUNTS):
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
is_failed = True
if is_failed:
break
time.sleep(RETRY_INTERVAL)
assert is_failed
assert path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file1)
for blk in blocks:
command = ["mv", blk + ".tmp", blk]
subprocess.check_output(command)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
# race condition: last restoration has failed
# but `isRestoring` hasn't been cleanup
for i in range(RETRY_COUNTS):
try:
restore_incrementally(dr_address, backup1, backup0_name)
break
except subprocess.CalledProcessError as e:
if "already in progress" not in e.output:
time.sleep(RETRY_INTERVAL)
else:
raise e
verify_no_frontend_data(grpc_engine_manager, 0, data1, grpc_dr_controller)
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
restore_incrementally(dr_address, backup2, backup1_name)
verify_no_frontend_data(grpc_engine_manager, 0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition
with pytest.raises(subprocess.CalledProcessError) as e:
restore_incrementally(dr_address, backup1, backup0_name)
assert "doesn't match lastRestored" in e
data3 = zero_string * length3 + data2[length3:length2]
restore_incrementally(dr_address, backup3, backup2_name)
verify_no_frontend_data(grpc_engine_manager, 0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# actually it is full restoration
restore_incrementally(dr_address, backup4, backup3_name)
verify_no_frontend_data(grpc_engine_manager, 0, snap4_data,
grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*tempoary"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_type(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
for backup_target in backup_targets:
address = grpc_controller.address
block_size = 2 * 1024 * 1024
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, block_size, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)
backup0_url = backup0["URL"]
assert backup0['IsIncremental'] is False
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)
backup1_url = backup1["URL"]
assert backup1['IsIncremental'] is True
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, block_size, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)
backup2_url = backup2["URL"]
assert backup2['IsIncremental'] is True
rm_backups(address, ENGINE_NAME, [backup2_url])
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, block_size, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)
backup3_url = backup3["URL"]
assert backup3['IsIncremental'] is False
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)
backup4_url = backup4["URL"]
assert backup4['IsIncremental'] is True
rm_backups(address, ENGINE_NAME,
[backup0_url, backup1_url, backup3_url, backup4_url])
cmd.sync_agent_server_reset(address)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
cleanup_controller(grpc_controller)
