def backup_test(
dev,
address, # NOQA
volume_name,
engine_name,
backup_target):
offset = 0
length = 128
snap1_data = random_string(length)
verify_data(dev, offset, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
backup1_info = create_backup(address, snap1, backup_target)
assert backup1_info["VolumeName"] == volume_name
assert backup1_info["Size"] == BLOCK_SIZE_STR
snap2_data = random_string(length)
verify_data(dev, offset, snap2_data)
snap2_checksum = checksum_dev(dev)
snap2 = cmd.snapshot_create(address)
backup2_info = create_backup(address, snap2, backup_target)
assert backup2_info["VolumeName"] == volume_name
assert backup2_info["Size"] == BLOCK_SIZE_STR
snap3_data = random_string(length)
verify_data(dev, offset, snap3_data)
snap3_checksum = checksum_dev(dev)
snap3 = cmd.snapshot_create(address)
backup3_info = create_backup(address, snap3, backup_target)
assert backup3_info["VolumeName"] == volume_name
assert backup3_info["Size"] == BLOCK_SIZE_STR
restore_with_frontend(address, engine_name, backup3_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap3_data
c = checksum_dev(dev)
assert c == snap3_checksum
rm_backups(address, engine_name, [backup3_info["URL"]])
restore_with_frontend(address, engine_name, backup1_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = checksum_dev(dev)
assert c == snap1_checksum
rm_backups(address, engine_name, [backup1_info["URL"]])
restore_with_frontend(address, engine_name, backup2_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap2_data
c = checksum_dev(dev)
assert c == snap2_checksum
rm_backups(address, engine_name, [backup2_info["URL"]])
def backup_hole_with_backing_file_test(backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_backing_dev(grpc_backing_replica1,
grpc_backing_replica2,
grpc_backing_controller)
offset1 = 512
length1 = 256
offset2 = 640
length2 = 256
boundary_offset = 0
boundary_length = 4100 # just pass 4096 into next 4k
hole_offset = 2 * 1024 * 1024
hole_length = 1024
snap1_data = random_string(length1)
verify_data(dev, offset1, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
boundary_data_backup1 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup1 = read_dev(dev, hole_offset, hole_length)
backup1_info = create_backup(address, snap1, backup_target)
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
snap2_checksum = checksum_dev(dev)
snap2 = cmd.snapshot_create(address)
boundary_data_backup2 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup2 = read_dev(dev, hole_offset, hole_length)
backup2_info = create_backup(address, snap2, backup_target)
restore_with_frontend(address, ENGINE_BACKING_NAME,
backup1_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup1
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup1
c = checksum_dev(dev)
assert c == snap1_checksum
restore_with_frontend(address, ENGINE_BACKING_NAME,
backup2_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup2
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup2
c = checksum_dev(dev)
assert c == snap2_checksum
def test_backup_corrupt_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
address = grpc_controller.address
length = 128
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
# write two backup blocks
verify_data(dev, 0, random_string(length))
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup1 = create_backup(address, snap, backup_target)
# overwrite second backup block
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup2 = create_backup(address, snap, backup_target)
# check that the volume now has 3 blocks
# backup1 and backup2 share the first block
# and have different second blocks
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# corrupt backup1 config
cfg = findfile(BACKUP_DIR, "backup_" + backup1["Name"] + ".cfg")
corrupt_backup = open(cfg, "w")
assert corrupt_backup
assert corrupt_backup.write("{corrupt: definitely") > 0
corrupt_backup.close()
cmd.backup_rm(address, backup1["URL"])
# check that the volume now has 2 blocks
# backup2 still relies on the backup1 first block
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 2)
# remove backup 2 and check that all blocks are deleted
cmd.backup_rm(address, backup2["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 0)
# remove volume.cfg then delete the backup volume
cfg = findfile(finddir(BACKUP_DIR, VOLUME_NAME), "volume.cfg")
os.remove(cfg)
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME,
backup_target)[VOLUME_NAME]
assert "cannot find" in info["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_volume_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
offset = 0
length = 128
address = grpc_controller.address
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
snap_data = random_string(length)
verify_data(dev, offset, snap_data)
snap = cmd.snapshot_create(address)
backup_info = create_backup(address, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
url = get_backup_volume_url(backup_target, VOLUME_NAME)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect_volume(address, url)
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def backup_with_backing_file_test(backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_backing_dev(grpc_backing_replica1,
grpc_backing_replica2,
grpc_backing_controller)
offset = 0
length = 256
snap0 = cmd.snapshot_create(address)
before = read_dev(dev, offset, length)
assert before != ""
snap0_checksum = checksum_dev(dev)
exists = read_from_backing_file(offset, length)
assert before == exists
backup0_info = create_backup(address, snap0, backup_target)
assert backup0_info["VolumeName"] == VOLUME_BACKING_NAME
backup_test(dev, address, VOLUME_BACKING_NAME,
ENGINE_BACKING_NAME, backup_target)
restore_with_frontend(address, ENGINE_BACKING_NAME,
backup0_info["URL"])
after = read_dev(dev, offset, length)
assert before == after
c = checksum_dev(dev)
assert c == snap0_checksum
rm_backups(address, ENGINE_BACKING_NAME, [backup0_info["URL"]])
def test_backup_volume_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
offset = 0
length = 128
address = grpc_controller.address
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
snap_data = random_string(length)
verify_data(dev, offset, snap_data)
snap = cmd.snapshot_create(address)
backup_info = create_backup(address, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME, backup_target)
assert "cannot find" in info[VOLUME_NAME]["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def snapshot_tree_backup_test(backup_target, engine_name, # NOQA
grpc_controller, grpc_replica1, grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2,
grpc_controller)
offset = 0
length = 128
backup = {}
snap, data = snapshot_tree_build(dev, address, engine_name,
offset, length)
backup["0b"] = create_backup(address, snap["0b"], backup_target)["URL"]
backup["0c"] = create_backup(address, snap["0c"], backup_target)["URL"]
backup["1c"] = create_backup(address, snap["1c"], backup_target)["URL"]
backup["2b"] = create_backup(address, snap["2b"], backup_target)["URL"]
backup["2c"] = create_backup(address, snap["2c"], backup_target)["URL"]
backup["3c"] = create_backup(address, snap["3c"], backup_target)["URL"]
snapshot_tree_verify_backup_node(dev, address, engine_name,
offset, length, backup, data, "0b")
snapshot_tree_verify_backup_node(dev, address, engine_name,
offset, length, backup, data, "0c")
snapshot_tree_verify_backup_node(dev, address, engine_name,
offset, length, backup, data, "1c")
snapshot_tree_verify_backup_node(dev, address, engine_name,
offset, length, backup, data, "2b")
snapshot_tree_verify_backup_node(dev, address, engine_name,
offset, length, backup, data, "2c")
snapshot_tree_verify_backup_node(dev, address, engine_name,
offset, length, backup, data, "3c")
def test_backup_block_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
address = grpc_controller.address
length = 128
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
# write two backup block
verify_data(dev, 0, random_string(length))
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup1 = create_backup(address, snap, backup_target)
assert backup1["VolumeName"] == VOLUME_NAME
assert backup1["Size"] == str(BLOCK_SIZE * 2)
assert snap in backup1["SnapshotName"]
# test block deduplication
backup1_duplicate = create_backup(address, snap, backup_target)
assert backup1_duplicate["VolumeName"] == VOLUME_NAME
assert backup1_duplicate["Size"] == str(BLOCK_SIZE * 2)
assert snap in backup1_duplicate["SnapshotName"]
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 2)
# overwrite second backup block
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup2 = create_backup(address, snap, backup_target)
assert backup2["VolumeName"] == VOLUME_NAME
assert backup2["Size"] == str(BLOCK_SIZE * 2)
assert snap in backup2["SnapshotName"]
# check that the volume now has 3 blocks
# backup1 and backup2 share the first block
# and have different second blocks
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# remove backup 1 duplicate
# this should not change the blocks on disk
# since all blocks are still required
cmd.backup_rm(address, backup1_duplicate["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# remove backup 1
# the volume should now have 2 blocks
# blk1 from backup1 should still be present
# since it's required by backup 2
cmd.backup_rm(address, backup1["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 2)
# remove the last remaining backup 2
# this should remove all blocks
cmd.backup_rm(address, backup2["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 0)
# cleanup the backup volume
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME,
backup_target)[VOLUME_NAME]
assert "cannot find" in info["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_incremental_logic(grpc_replica1, grpc_replica2,
grpc_controller, backup_targets): # NOQA
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
address = grpc_controller.address
volume_name = VOLUME_NAME
engine_name = ENGINE_NAME
offset = 0
length = 128
# initial backup
snap1_data = random_string(length)
verify_data(dev, offset, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
backup1_info = create_backup(address, snap1, backup_target)
assert backup1_info["IsIncremental"] is False
# delta backup on top of initial backup
snap2_data = random_string(int(length / 2))
verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2_info = create_backup(address, snap2, backup_target)
assert backup2_info["IsIncremental"] is True
# delete the volume
cmd.sync_agent_server_reset(address)
grpc_controller = cleanup_controller(grpc_controller)
grpc_replica1 = cleanup_replica(grpc_replica1)
grpc_replica2 = cleanup_replica(grpc_replica2)
# recreate the volume
dev = get_dev(grpc_replica1,
grpc_replica2,
grpc_controller,
clean_backup_dir=False)
# empty initial backup after volume recreation
snap3 = cmd.snapshot_create(address)
backup3_info = create_backup(address, snap3, backup_target)
assert backup3_info["VolumeName"] == volume_name
assert backup3_info["Size"] == '0'
assert backup3_info["IsIncremental"] is False
# write half of snap1 onto head
snap4_data = snap1_data[:int(length / 2)]
assert len(snap4_data) == int(length / 2)
verify_data(dev, offset, snap4_data)
snap4_checksum = checksum_dev(dev)
assert snap4_checksum != snap1_checksum
snap4 = cmd.snapshot_create(address)
backup4_info = create_backup(address, snap4, backup_target)
assert backup4_info["IsIncremental"] is True
# restore initial backup
reset_volume(grpc_controller, grpc_replica1, grpc_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, engine_name, backup1_info["URL"])
assert read_dev(dev, offset, length) == snap1_data
assert checksum_dev(dev) == snap1_checksum
# restore final backup (half of snap1)
reset_volume(grpc_controller, grpc_replica1, grpc_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, engine_name, backup4_info["URL"])
assert checksum_dev(dev) == snap4_checksum
assert snap4_checksum != snap1_checksum
data = read_dev(dev, offset, length)
assert data[:int(length / 2)] == snap4_data
assert data[int(length / 2):] == '\x00' * int(length / 2)
rm_backups(address, engine_name, [
backup1_info["URL"], backup2_info["URL"], backup3_info["URL"],
backup4_info["URL"]
])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_S3_latest_unavailable(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
for backup_target in backup_targets:
if "s3://" not in backup_target:
continue
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
address = grpc_controller.address
volume_name = VOLUME_NAME
engine_name = ENGINE_NAME
offset = 0
length = 128
# initial backup
snap1_data = random_string(length)
verify_data(dev, offset, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
backup1_info = create_backup(address, snap1, backup_target)
# backup to be unavailable
snap2_data = random_string(length)
verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2_info = create_backup(address, snap2, backup_target)
# the gc after the restore will clean up the missing backup
cfg = findfile(BACKUP_DIR, "backup_" + backup2_info["Name"] + ".cfg")
os.remove(cfg)
# final full backup after unavailable backup
snap3_data = random_string(length)
verify_data(dev, offset, snap3_data)
snap3_checksum = checksum_dev(dev)
snap3 = cmd.snapshot_create(address)
backup3_info = create_backup(address, snap3, backup_target)
assert backup3_info["VolumeName"] == volume_name
assert backup3_info["Size"] == BLOCK_SIZE_STR
# write some stuff on head
head_data = random_string(length)
verify_data(dev, offset, head_data)
# test restore of the initial backup
reset_volume(grpc_controller, grpc_replica1, grpc_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, engine_name, backup1_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = checksum_dev(dev)
assert c == snap1_checksum
# test a restore for the final backup
reset_volume(grpc_controller, grpc_replica1, grpc_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, engine_name, backup3_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap3_data
c = checksum_dev(dev)
assert c == snap3_checksum
rm_backups(address, engine_name,
[backup1_info["URL"], backup3_info["URL"]])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def restore_inc_test(grpc_engine_manager, # NOQA
grpc_controller, # NOQA
grpc_replica1, grpc_replica2, # NOQA
grpc_dr_controller, # NOQA
grpc_dr_replica1, grpc_dr_replica2, # NOQA
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_engine_manager,
grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(grpc_engine_manager,
0, snap0_data, grpc_dr_controller)
# mock restore crash/error
delta_file1 = "volume-delta-" + backup0_name + ".img"
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk+".tmp".encode('utf-8')]
subprocess.check_output(command).strip()
# should fail
is_failed = False
cmd.restore_inc(dr_address, backup1, backup0_name)
for i in range(RETRY_COUNTS):
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
is_failed = True
if is_failed:
break
time.sleep(RETRY_INTERVAL)
assert is_failed
assert path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file1)
for blk in blocks:
command = ["mv", blk+".tmp".encode('utf-8'), blk]
subprocess.check_output(command)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
# race condition: last restoration has failed
# but `isRestoring` hasn't been cleanup
for i in range(RETRY_COUNTS):
try:
restore_incrementally(dr_address, backup1, backup0_name)
break
except subprocess.CalledProcessError as e:
if "already in progress" not in e.output:
time.sleep(RETRY_INTERVAL)
else:
raise e
verify_no_frontend_data(grpc_engine_manager,
0, data1, grpc_dr_controller)
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
restore_incrementally(dr_address, backup2, backup1_name)
verify_no_frontend_data(grpc_engine_manager,
0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition
with pytest.raises(subprocess.CalledProcessError) as e:
restore_incrementally(dr_address, backup1, backup0_name)
assert "doesn't match lastRestored" in e
data3 = zero_string * length3 + data2[length3:length2]
restore_incrementally(dr_address, backup3, backup2_name)
verify_no_frontend_data(grpc_engine_manager,
0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# actually it is full restoration
restore_incrementally(dr_address, backup4, backup3_name)
verify_no_frontend_data(grpc_engine_manager,
0, snap4_data, grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*tempoary"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_engine_manager,
grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def restore_to_file_without_backing_file_test(
backup_target, # NOQA
grpc_controller, # NOQA
grpc_replica1, # NOQA
grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
length0 = 256
length1 = 128
offset0 = 0
offset1 = length1 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot for converting to init state.
snap0 = cmd.snapshot_create(address)
# create 1 snapshot with 256B data.
# output = snap2(offset0, length1)
snap1_data = random_string(length0)
verify_data(dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
backup = create_backup(address, snap1, backup_target)["URL"]
cmd.restore_to_file(address, backup, "", output_raw_path, IMAGE_FORMAT_RAW)
output1_raw = read_file(output_raw_path, offset0, length0)
assert output1_raw == snap1_data
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, "", output_qcow2_path,
IMAGE_FORMAT_QCOW2)
output1_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
assert output1_qcow2.decode('utf-8') == snap1_data
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_NAME, snap0)
rm_snaps(address, [snap1])
rm_backups(address, ENGINE_NAME, [backup])
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2)
snap1_data = random_string(length0)
verify_data(dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
snap2_data = random_string(length1)
verify_data(dev, offset0, snap2_data)
snap2 = cmd.snapshot_create(address)
backup = create_backup(address, snap2, backup_target)["URL"]
cmd.restore_to_file(address, backup, "", output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(output_raw_path, offset0, length1)
output2_raw_snap1 = read_file(output_raw_path, offset1, length0 - length1)
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset1:length0]
cmd.restore_to_file(address, backup, "", output_qcow2_path,
IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - length1)
assert output2_qcow2_snap2.decode('utf-8') == snap2_data
assert output2_qcow2_snap1.decode('utf-8') == snap1_data[offset1:length0]
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_NAME, snap0)
rm_snaps(address, [snap1, snap2])
rm_backups(address, ENGINE_NAME, [backup])
def test_backup_lock(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
"""
Test backup locks
Context:
The idea is to implement a locking mechanism that utilizes the backupstore,
to prevent the following dangerous cases of concurrent operations.
- prevent backup deletion during backup restoration
- prevent backup deletion while a backup is in progress
- prevent backup creation during backup deletion
- prevent backup restoration during backup deletion
Steps:
1. Create a volume(1) and attach to the current node
2. create a backup(1) of volume(1)
3. verify backup(1) creation completed
4. write some data to volume(1)
5. create an active lock of type Delete
6. create a backup(2) of volume(1)
7. verify backup(2) creation timed out
8. delete active lock of type Delete
9. create an active lock of type Delete
10. restore backup(1)
11. verify backup(1) restore timed out
12. delete active lock of type Delete
13. restore backup(1)
14. verify backup(1) restore completed
15. create an active lock of type Restore
16. delete backup(1)
17. verify backup(1) deletion timed out
18. delete active lock of type Restore
19. delete backup(1)
20. verify backup(1) deletion completed
21. cleanup
"""
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
# create a regular backup
address = grpc_controller.address
offset = 0
length = 128
snap1_data = random_string(length)
verify_data(dev, offset, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
# create a backup to create the volume
info = create_backup(address, snap1, backup_target)
assert info["VolumeName"] == VOLUME_NAME
assert info["Size"] == BLOCK_SIZE_STR
assert snap1 in info["SnapshotName"]
# backup should error out with timeout
# because of delete lock
create_delete_lock(True)
with pytest.raises(subprocess.CalledProcessError):
create_backup(address, snap1, backup_target)
remove_lock_file(DELETE_LOCK)
# restore should error out with timeout
# because of delete lock
create_delete_lock(True)
with pytest.raises(subprocess.CalledProcessError):
restore_with_frontend(address, ENGINE_NAME, info["URL"])
remove_lock_file(DELETE_LOCK)
# restore should succeed now, that there is no active delete lock
restore_with_frontend(address, ENGINE_NAME, info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = checksum_dev(dev)
assert c == snap1_checksum
# delete should error out with timeout
# because of restore lock
create_restore_lock(True)
with pytest.raises(subprocess.CalledProcessError):
rm_backups(address, ENGINE_NAME, [info["URL"]])
remove_lock_file(RESTORE_LOCK)
# delete should succeed now, that there is no active restore lock
rm_backups(address, ENGINE_NAME, [info["URL"]])
# cleanup volume 1
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_inc_restore_with_rebuild_and_expansion(grpc_controller, grpc_replica1,
grpc_replica2,
grpc_controller_no_frontend,
grpc_fixed_dir_replica1,
grpc_fixed_dir_replica2,
backup_targets): # NOQA
# Pick up a random backup target.
backup_target = backup_targets[random.randint(0, 1)]
address = grpc_controller.address
dr_address = grpc_controller_no_frontend.address
try:
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
except Exception:
pass
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
start_no_frontend_volume(grpc_controller_no_frontend,
grpc_fixed_dir_replica1)
data0_len = random_length(PAGE_SIZE)
data0 = Data(random.randrange(0, SIZE - 2 * PAGE_SIZE, PAGE_SIZE),
data0_len, random_string(data0_len))
snap0 = Snapshot(dev, data0, address)
backup0_info = create_backup(address, snap0.name, backup_target)
assert backup0_info["VolumeName"] == VOLUME_NAME
assert backup0_info["Size"] == str(BLOCK_SIZE)
cmd.backup_restore(dr_address, backup0_info["URL"])
wait_for_restore_completion(dr_address, backup0_info["URL"])
verify_no_frontend_data(data0.offset, data0.content,
grpc_controller_no_frontend)
expand_volume_with_frontend(grpc_controller, EXPANDED_SIZE)
wait_and_check_volume_expansion(grpc_controller, EXPANDED_SIZE)
data1_len = random_length(PAGE_SIZE)
data1 = Data(random.randrange(SIZE, EXPANDED_SIZE - PAGE_SIZE, PAGE_SIZE),
data1_len, random_string(data1_len))
snap1 = Snapshot(dev, data1, address)
backup1_info = create_backup(address, snap1.name, backup_target,
EXPANDED_SIZE_STR)
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["Size"] == str(2 * BLOCK_SIZE)
backup_volumes = cmd.backup_volume_list(address, VOLUME_NAME,
backup_target)
assert VOLUME_NAME in backup_volumes
url = get_backup_volume_url(backup_target, VOLUME_NAME)
backup_info = cmd.backup_inspect_volume(address, url)
assert backup_info["Size"] == EXPANDED_SIZE_STR
# restore command invocation should error out
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup1_info["URL"])
assert "need to expand the DR volume" in e.value.stdout
# The above restore error is triggered before calling the replicas.
# Hence the error won't be recorded in the restore status
# and we can continue restoring backups for the DR volume.
rs = cmd.restore_status(dr_address)
for status in rs.values():
assert status['backupURL'] == backup0_info["URL"]
assert status['lastRestored'] == backup0_info["Name"]
assert 'error' not in status.keys()
assert not status["isRestoring"]
grpc_controller_no_frontend.volume_expand(EXPANDED_SIZE)
wait_for_volume_expansion(grpc_controller_no_frontend, EXPANDED_SIZE)
# This restore command will trigger snapshot purge.
# And the error is triggered before calling the replicas.
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup1_info["URL"])
assert "found more than 1 snapshot in the replicas, " \
"hence started to purge snapshots before the restore" \
in e.value.stdout
wait_for_purge_completion(dr_address)
snaps_info = cmd.snapshot_info(dr_address)
assert len(snaps_info) == 2
volume_head_name = "volume-head"
snap_name = "expand-" + EXPANDED_SIZE_STR
head_info = snaps_info[volume_head_name]
assert head_info["name"] == volume_head_name
assert head_info["parent"] == snap_name
assert not head_info["children"]
assert head_info["usercreated"] is False
snap_info = snaps_info[snap_name]
assert snap_info["name"] == snap_name
assert not snap_info["parent"]
assert volume_head_name in snap_info["children"]
assert snap_info["usercreated"] is False
cmd.backup_restore(dr_address, backup1_info["URL"])
wait_for_restore_completion(dr_address, backup1_info["URL"])
verify_no_frontend_data(data1.offset, data1.content,
grpc_controller_no_frontend)
# For DR volume, the rebuilding replica won't be expanded automatically.
open_replica(grpc_fixed_dir_replica2)
with pytest.raises(subprocess.CalledProcessError):
cmd.add_replica(dr_address, grpc_fixed_dir_replica2.url, True)
# Manually expand the rebuilding replica then retry `add-replica`.
grpc_fixed_dir_replica2.replica_open()
grpc_fixed_dir_replica2.replica_expand(EXPANDED_SIZE)
grpc_fixed_dir_replica2.replica_close()
cmd.add_replica(dr_address, grpc_fixed_dir_replica2.url, True)
replicas = grpc_controller_no_frontend.replica_list()
assert len(replicas) == 2
rw_replica, wo_replica = 0, 0
for r in replicas:
if r.mode == 'RW':
rw_replica += 1
else:
assert r.mode == "WO"
wo_replica += 1
assert rw_replica == 1 and wo_replica == 1
# The old replica will fail the restore but the error won't be recorded.
# Then rebuilding replica will start full restore.
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup1_info["URL"])
assert "already restored backup" in e.value.stdout
wait_for_restore_completion(dr_address, backup1_info["URL"])
cmd.verify_rebuild_replica(dr_address, grpc_fixed_dir_replica2.url)
replicas = grpc_controller_no_frontend.replica_list()
assert len(replicas) == 2
for r in replicas:
assert r.mode == 'RW'
verify_no_frontend_data(data1.offset, data1.content,
grpc_controller_no_frontend)
cmd.backup_volume_rm(grpc_controller.address, VOLUME_NAME, backup_target)
def restore_inc_test(grpc_controller, grpc_replica1, grpc_replica2,
grpc_dr_controller, grpc_dr_replica1, grpc_dr_replica2,
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
# mock restore crash/error:
# By adding attribute `immutable`, Longhorn cannot create a file
# for the restore. Then the following restore command will fail.
command = ["chattr", "+i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "+i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup0)
assert "operation not permitted" in e.value.stdout
# the restore status will be reverted/keep unchanged
# if an error is triggered before the actual restore is performed
rs = cmd.restore_status(dr_address)
for status in rs.values():
assert not status["isRestoring"]
assert not status['backupURL']
assert 'error' not in status
assert 'progress' not in status
assert not status['state']
command = ["chattr", "-i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "-i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
cmd.sync_agent_server_reset(dr_address)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(0, snap0_data, grpc_dr_controller)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
cmd.backup_restore(dr_address, backup1)
wait_for_restore_completion(dr_address, backup1)
verify_no_frontend_data(0, data1, grpc_dr_controller)
delta_file1 = "volume-delta-" + backup0_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
cmd.backup_restore(dr_address, backup2)
wait_for_restore_completion(dr_address, backup2)
verify_no_frontend_data(0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition: duplicate inc restore calls
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup2)
wait_for_restore_completion(dr_address, backup2)
assert "already restored backup" in e.value.stdout
data3 = zero_string * length3 + data2[length3:length2]
cmd.backup_restore(dr_address, backup3)
wait_for_restore_completion(dr_address, backup3)
verify_no_frontend_data(0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# This inc restore will fall back to full restore
cmd.backup_restore(dr_address, backup4)
wait_for_restore_completion(dr_address, backup4)
verify_no_frontend_data(0, snap4_data, grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
# check if the tmp files during this special full restore are cleaned up.
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*snap_tmp"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cmd.sync_agent_server_reset(dr_address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def volume_expansion_with_backup_test(
grpc_engine_manager, # NOQA
grpc_controller, # NOQA
grpc_dr_controller, # NOQA
grpc_replica1,
grpc_replica2, # NOQA
grpc_dr_replica1, # NOQA
grpc_dr_replica2, # NOQA
volume_name,
engine_name,
backup_target): # NOQA
address = grpc_controller.address
dr_address = grpc_dr_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
start_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
try:
cmd.backup_volume_rm(address, volume_name, backup_target)
except Exception:
pass
data0_len = random_length(PAGE_SIZE)
data0 = Data(random.randrange(0, SIZE - 2 * PAGE_SIZE, PAGE_SIZE),
data0_len, random_string(data0_len))
snap0 = Snapshot(dev, data0, address)
backup0_info = create_backup(address, snap0.name, backup_target)
assert backup0_info["VolumeName"] == volume_name
assert backup0_info["Size"] == str(BLOCK_SIZE)
cmd.backup_restore(dr_address, backup0_info["URL"])
wait_for_restore_completion(dr_address, backup0_info["URL"])
verify_no_frontend_data(grpc_engine_manager, data0.offset, data0.content,
grpc_dr_controller)
expand_volume_with_frontend(grpc_controller, EXPANDED_SIZE)
wait_and_check_volume_expansion(grpc_controller, EXPANDED_SIZE)
data1_len = random_length(PAGE_SIZE)
data1 = Data(random.randrange(SIZE, EXPANDED_SIZE - PAGE_SIZE, PAGE_SIZE),
data1_len, random_string(data1_len))
snap1 = Snapshot(dev, data1, address)
backup1_info = create_backup(address, snap1.name, backup_target,
EXPANDED_SIZE_STR)
assert backup1_info["VolumeName"] == volume_name
assert backup1_info["Size"] == str(2 * BLOCK_SIZE)
backup_volumes = cmd.backup_volume_list(address, volume_name,
backup_target)
assert volume_name in backup_volumes
assert backup_volumes[volume_name]["Size"] == EXPANDED_SIZE_STR
# incremental restoration will implicitly expand the volume first
restore_incrementally(dr_address, backup1_info["URL"],
backup0_info["Name"])
check_dr_volume_block_device_size(grpc_engine_manager, grpc_dr_controller,
EXPANDED_SIZE)
verify_no_frontend_data(grpc_engine_manager, data1.offset, data1.content,
grpc_dr_controller)
cmd.backup_volume_rm(grpc_controller.address, volume_name, backup_target)
def restore_to_file_with_backing_file_test(
backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
backing_dev = get_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
length0 = 4 * 1024
length1 = 256
length2 = 128
offset0 = 0
offset1 = length1 + offset0
offset2 = length2 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot.
# data in output image == data in backing
check_backing()
check_empty_volume(backing_dev)
snap0 = cmd.snapshot_create(address)
backup = create_backup(address, snap0, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset0, length0)
dev_checksum = checksum_dev(backing_dev)
assert volume_data != ""
assert volume_data == backing_data
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output0_raw = read_file(output_raw_path, offset0, length0)
output0_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE).encode('utf-8'))
assert output0_raw == backing_data
assert output0_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output0_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
output0_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output0_qcow2.decode('utf-8') == backing_data
assert output0_qcow2.decode('utf-8') == volume_data
assert output0_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
rm_backups(address, ENGINE_BACKING_NAME, [backup])
# create 1 snapshot with 256B data.
# output = snap1(offset0, length1) + backing(offset1, ...)
snap1_data = random_string(length1)
verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
backup = create_backup(address, snap1, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset1, length0 - offset1)
dev_checksum = checksum_dev(backing_dev)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output1_raw_snap1 = read_file(output_raw_path, offset0, length1)
output1_raw_backing = read_file(output_raw_path, offset1,
length0 - offset1)
output1_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE).encode('utf-8'))
assert output1_raw_snap1 == snap1_data
assert output1_raw_backing == backing_data
assert output1_raw_snap1 + output1_raw_backing == volume_data
assert output1_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output1_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output1_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output1_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output1_qcow2_snap1.decode('utf-8') == snap1_data
assert output1_qcow2_backing.decode('utf-8') == backing_data
assert output1_qcow2_snap1.decode('utf-8') + \
output1_qcow2_backing.decode('utf-8') == volume_data
assert output1_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
rm_snaps(address, [snap1])
rm_backups(address, ENGINE_BACKING_NAME, [backup])
check_backing()
check_empty_volume(backing_dev)
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2) + backing(offset2, ...)
snap1_data = random_string(length1)
verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
snap2_data = random_string(length2)
verify_data(backing_dev, offset0, snap2_data)
snap2 = cmd.snapshot_create(address)
backup = create_backup(address, snap2, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset1, length0 - offset1)
dev_checksum = checksum_dev(backing_dev)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(output_raw_path, offset0, length2)
output2_raw_snap1 = read_file(output_raw_path, offset2, length1 - length2)
output2_raw_backing = read_file(output_raw_path, offset1,
length0 - offset1)
output2_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE).encode('utf-8'))
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset2:length1]
assert output2_raw_backing == backing_data
assert \
volume_data == \
output2_raw_snap2 + output2_raw_snap1 + output2_raw_backing
assert output2_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length2)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset2, length1 - length2)
output2_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output2_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output2_qcow2_snap2.decode('utf-8') == snap2_data
assert output2_qcow2_snap1.decode('utf-8') == snap1_data[offset2:length1]
assert output2_qcow2_backing.decode('utf-8') == backing_data
assert \
volume_data == \
output2_qcow2_snap2.decode('utf-8') + \
output2_qcow2_snap1.decode('utf-8') + \
output1_qcow2_backing.decode('utf-8')
assert output2_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
rm_snaps(address, [snap1, snap2])
rm_backups(address, ENGINE_BACKING_NAME, [backup])
check_backing()
check_empty_volume(backing_dev)
def test_backup_block_no_cleanup(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
address = grpc_controller.address
length = 128
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
# write two backup blocks
verify_data(dev, 0, random_string(length))
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup1 = create_backup(address, snap, backup_target)
assert backup1["VolumeName"] == VOLUME_NAME
assert backup1["Size"] == str(BLOCK_SIZE * 2)
assert snap in backup1["SnapshotName"]
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 2)
# overwrite second backup block
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup2 = create_backup(address, snap, backup_target)
assert backup2["VolumeName"] == VOLUME_NAME
assert backup2["Size"] == str(BLOCK_SIZE * 2)
assert snap in backup2["SnapshotName"]
# check that the volume now has 3 blocks
# backup1 and backup2 share the first block
# and have different second blocks
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# create an artificial in progress backup
# that will stop the gc from removing blocks
in_progress_backup_file = create_in_progress_backup_file(VOLUME_NAME)
# remove backup 1 the volume should still have 3 blocks
cmd.backup_rm(address, backup1["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# remove the in progress backup
os.remove(in_progress_backup_file)
# remove the last remaining backup 2
# this should remove all blocks
# including the orphaned block from backup 1
cmd.backup_rm(address, backup2["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 0)
# cleanup the backup volume
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME,
backup_target)[VOLUME_NAME]
assert "cannot find" in info["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def backup_hole_with_backing_file_test(
backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
volume_name = grpc_backing_controller.volume_get().name
assert volume_name == VOLUME_BACKING_NAME
offset1 = 512
length1 = 256
offset2 = 640
length2 = 256
boundary_offset = 0
boundary_length = 4100 # just pass 4096 into next 4k
hole_offset = 2 * 1024 * 1024
hole_length = 1024
snap1_data = random_string(length1)
verify_data(dev, offset1, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
boundary_data_backup1 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup1 = read_dev(dev, hole_offset, hole_length)
backup1_info = create_backup(address,
snap1,
backup_target,
backing_image_name=BACKING_IMAGE_NAME,
backing_image_url=BACKING_IMAGE_URL)
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
snap2_checksum = checksum_dev(dev)
snap2 = cmd.snapshot_create(address)
boundary_data_backup2 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup2 = read_dev(dev, hole_offset, hole_length)
backup2_info = create_backup(address,
snap2,
backup_target,
backing_image_name=BACKING_IMAGE_NAME,
backing_image_url=BACKING_IMAGE_URL)
assert backup2_info["VolumeBackingImageName"] == BACKING_IMAGE_NAME
assert backup2_info["VolumeBackingImageURL"] == BACKING_IMAGE_URL
reset_volume(grpc_backing_controller, grpc_backing_replica1,
grpc_backing_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, ENGINE_BACKING_NAME, backup1_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup1
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup1
c = checksum_dev(dev)
assert c == snap1_checksum
reset_volume(grpc_backing_controller, grpc_backing_replica1,
grpc_backing_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, ENGINE_BACKING_NAME, backup2_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup2
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup2
c = checksum_dev(dev)
assert c == snap2_checksum
def test_restore_with_rebuild(grpc_controller, grpc_replica1, grpc_replica2,
grpc_controller_no_frontend,
grpc_fixed_dir_replica1, grpc_fixed_dir_replica2,
backup_targets): # NOQA
# Pick up a random backup target.
backup_target = backup_targets[random.randint(0, 1)]
address = grpc_controller.address
dr_address = grpc_controller_no_frontend.address
try:
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
except Exception:
pass
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
start_no_frontend_volume(grpc_controller_no_frontend,
grpc_fixed_dir_replica1)
data0_len = random_length(PAGE_SIZE)
data0 = Data(random.randrange(0, SIZE - 2 * PAGE_SIZE, PAGE_SIZE),
data0_len, random_string(data0_len))
snap0 = Snapshot(dev, data0, address)
backup0_info = create_backup(address, snap0.name, backup_target)
assert backup0_info["VolumeName"] == VOLUME_NAME
assert backup0_info["Size"] == str(BLOCK_SIZE)
cmd.backup_restore(dr_address, backup0_info["URL"])
wait_for_restore_completion(dr_address, backup0_info["URL"])
verify_no_frontend_data(data0.offset, data0.content,
grpc_controller_no_frontend)
open_replica(grpc_fixed_dir_replica2)
cmd.add_replica(dr_address, grpc_fixed_dir_replica2.url, True)
replicas = grpc_controller_no_frontend.replica_list()
assert len(replicas) == 2
rw_replica, wo_replica = 0, 0
for r in replicas:
if r.mode == 'RW':
rw_replica += 1
else:
assert r.mode == "WO"
wo_replica += 1
assert rw_replica == 1 and wo_replica == 1
# The old replica will fail the restore but the error won't be recorded.
# Then rebuilding replica will start full restore.
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup0_info["URL"])
assert "already restored backup" in e.value.stdout
wait_for_restore_completion(dr_address, backup0_info["URL"])
# Need to manually verify the rebuilding replica for the restore volume
cmd.verify_rebuild_replica(dr_address, grpc_fixed_dir_replica2.url)
replicas = grpc_controller_no_frontend.replica_list()
assert len(replicas) == 2
for r in replicas:
assert r.mode == 'RW'
# Delete the old replica then check if the rebuilt replica works fine.
cleanup_replica(grpc_fixed_dir_replica1)
grpc_controller_no_frontend.replica_delete(grpc_fixed_dir_replica1.address)
verify_no_frontend_data(data0.offset, data0.content,
grpc_controller_no_frontend)
cmd.backup_volume_rm(grpc_controller.address, VOLUME_NAME, backup_target)
def restore_inc_test(
grpc_engine_manager, # NOQA
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2, # NOQA
grpc_dr_controller, # NOQA
grpc_dr_replica1,
grpc_dr_replica2, # NOQA
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
# mock restore crash/error:
# By adding attribute `immutable`, Longhorn cannot create a file
# for the restore. Then the following restore command will fail.
command = ["chattr", "+i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "+i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup0)
assert "operation not permitted" in e.value.stdout
command = ["chattr", "-i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "-i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(grpc_engine_manager, 0, snap0_data,
grpc_dr_controller)
delta_file1 = "volume-delta-" + backup0_name + ".img"
# mock inc restore crash/error: cannot clean up the delta file path
command = ["mkdir", "-p", FIXED_REPLICA_PATH1 + delta_file1 + "/dir"]
subprocess.check_output(command).strip()
command = ["mkdir", "-p", FIXED_REPLICA_PATH2 + delta_file1 + "/dir"]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError):
cmd.restore_inc(dr_address, backup1, backup0_name)
command = ["rm", "-r", FIXED_REPLICA_PATH1 + delta_file1]
subprocess.check_output(command).strip()
command = ["rm", "-r", FIXED_REPLICA_PATH2 + delta_file1]
subprocess.check_output(command).strip()
if "vfs" in backup_target:
# restore status should contain the error info
failed_restore, finished_restore = 0, 0
for i in range(RETRY_COUNTS):
failed_restore, finished_restore = 0, 0
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'failed to clean up the existing file' in\
status['error']
failed_restore += 1
if not status["isRestoring"]:
finished_restore += 1
if failed_restore == 2 and finished_restore == 2:
break
time.sleep(RETRY_INTERVAL)
assert failed_restore == 2 and finished_restore == 2
# mock inc restore error: invalid block
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk + ".tmp".encode('utf-8')]
subprocess.check_output(command).strip()
cmd.restore_inc(dr_address, backup1, backup0_name)
# restore status should contain the error info
failed_restore, finished_restore = 0, 0
for i in range(RETRY_COUNTS):
failed_restore, finished_restore = 0, 0
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
failed_restore += 1
if not status["isRestoring"]:
finished_restore += 1
if failed_restore == 2 and finished_restore == 2:
break
time.sleep(RETRY_INTERVAL)
assert failed_restore == 2 and finished_restore == 2
assert path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file1)
for blk in blocks:
command = ["mv", blk + ".tmp".encode('utf-8'), blk]
subprocess.check_output(command)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
restore_incrementally(dr_address, backup1, backup0_name)
verify_no_frontend_data(grpc_engine_manager, 0, data1, grpc_dr_controller)
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
restore_incrementally(dr_address, backup2, backup1_name)
verify_no_frontend_data(grpc_engine_manager, 0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition
with pytest.raises(subprocess.CalledProcessError) as e:
restore_incrementally(dr_address, backup1, backup0_name)
assert "doesn't match field LastRestored" in e.value.stdout
data3 = zero_string * length3 + data2[length3:length2]
restore_incrementally(dr_address, backup3, backup2_name)
verify_no_frontend_data(grpc_engine_manager, 0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# actually it is full restoration
restore_incrementally(dr_address, backup4, backup3_name)
verify_no_frontend_data(grpc_engine_manager, 0, snap4_data,
grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*tempoary"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_volume_list(
grpc_replica_client,
grpc_controller_client, # NOQA
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
"""
Test backup volume list
Context:
We want to make sure that an error when listing a single backup volume
does not stop us from listing all the other backup volumes. Otherwise a
single faulty backup can block the retrieval of all known backup volumes.
Steps:
1. Create a volume(1,2) and attach to the current node
2. write some data to volume(1,2)
3. Create a backup(1) of volume(1,2)
4. request a backup list
5. verify backup list contains no error messages for volume(1,2)
6. verify backup list contains backup(1) for volume(1,2)
7. place a file named "*****@*****.**"
into the backups folder of volume(1)
8. request a backup list
9. verify backup list contains no error messages for volume(1,2)
10. verify backup list contains backup(1) for volume(1,2)
11. delete backup volumes(1 & 2)
12. cleanup
"""
# create a second volume
grpc2_replica1 = grpc_replica_client(REPLICA_2_NAME + "-1")
grpc2_replica2 = grpc_replica_client(REPLICA_2_NAME + "-2")
grpc2_controller = grpc_controller_client(ENGINE2_NAME, VOLUME2_NAME)
offset = 0
length = 128
address = grpc_controller.address
address2 = grpc2_controller.address
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
dev2 = get_dev(grpc2_replica1, grpc2_replica2, grpc2_controller)
# create a regular backup
snap_data = random_string(length)
verify_data(dev, offset, snap_data)
snap = cmd.snapshot_create(address)
backup_info = create_backup(address, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
# create a regular backup on volume 2
verify_data(dev2, offset, random_string(length))
snap = cmd.snapshot_create(address2)
backup_info = create_backup(address2, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME2_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
# request a volume list
info = cmd.backup_volume_list(address,
"",
backup_target,
include_backup_details=True)
assert info[VOLUME_NAME]["Name"] == VOLUME_NAME
assert len(info[VOLUME_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME_NAME]["Messages"]
assert info[VOLUME2_NAME]["Name"] == VOLUME2_NAME
assert len(info[VOLUME2_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME2_NAME]["Messages"]
# place badly named backup.cfg file
# we want the list call to return all valid files correctly
backup_dir = os.path.join(finddir(BACKUP_DIR, VOLUME_NAME), "backups")
cfg = open(os.path.join(backup_dir, "*****@*****.**"), "w")
cfg.close()
info = cmd.backup_volume_list(address,
"",
backup_target,
include_backup_details=True)
assert info[VOLUME_NAME]["Name"] == VOLUME_NAME
assert len(info[VOLUME_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME_NAME]["Messages"]
assert info[VOLUME2_NAME]["Name"] == VOLUME2_NAME
assert len(info[VOLUME2_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME2_NAME]["Messages"]
# remove the volume with the badly named backup.cfg
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address,
VOLUME_NAME,
backup_target,
include_backup_details=True)
assert "cannot find" in info[VOLUME_NAME]["Messages"]["error"]
# remove volume 2 backups
cmd.backup_volume_rm(address, VOLUME2_NAME, backup_target)
info = cmd.backup_volume_list(address,
VOLUME2_NAME,
backup_target,
include_backup_details=True)
assert "cannot find" in info[VOLUME2_NAME]["Messages"]["error"]
# cleanup volume 1
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
# cleanup volume 2
cmd.sync_agent_server_reset(address2)
cleanup_controller(grpc2_controller)
cleanup_replica(grpc2_replica1)
cleanup_replica(grpc2_replica2)
def inc_restore_failure_cleanup_error_test(grpc_controller, grpc_replica1,
grpc_replica2, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2,
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(0, snap0_data, grpc_dr_controller)
# mock inc restore crash/error: cannot clean up the delta file path
delta_file = "volume-delta-" + backup0_name + ".img"
command = ["mkdir", "-p", FIXED_REPLICA_PATH1 + delta_file + "/dir"]
subprocess.check_output(command).strip()
command = ["mkdir", "-p", FIXED_REPLICA_PATH2 + delta_file + "/dir"]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup1)
assert "failed to clean up the existing file" in e.value.stdout
command = ["rm", "-r", FIXED_REPLICA_PATH1 + delta_file]
subprocess.check_output(command).strip()
command = ["rm", "-r", FIXED_REPLICA_PATH2 + delta_file]
subprocess.check_output(command).strip()
# the restore status will be reverted/keep unchanged
# if an error is triggered before the actual restore is performed
rs = cmd.restore_status(dr_address)
for status in rs.values():
assert not status["isRestoring"]
assert status['backupURL'] == backup0
assert 'error' not in status
assert status['progress'] == 100
assert status['state'] == "complete"
cleanup_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1])
cmd.sync_agent_server_reset(address)
cmd.sync_agent_server_reset(dr_address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_type(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
for backup_target in backup_targets:
address = grpc_controller.address
block_size = 2 * 1024 * 1024
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, block_size, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)
backup0_url = backup0["URL"]
assert backup0['IsIncremental'] is False
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)
backup1_url = backup1["URL"]
assert backup1['IsIncremental'] is True
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, block_size, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)
backup2_url = backup2["URL"]
assert backup2['IsIncremental'] is True
rm_backups(address, ENGINE_NAME, [backup2_url])
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, block_size, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)
backup3_url = backup3["URL"]
assert backup3['IsIncremental'] is True
# full backup: backup the same snapshot twice
backup3 = create_backup(address, snap3, backup_target)
backup3_url = backup3["URL"]
assert backup3['IsIncremental'] is False
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)
backup4_url = backup4["URL"]
assert backup4['IsIncremental'] is True
rm_backups(address, ENGINE_NAME,
[backup0_url, backup1_url, backup3_url, backup4_url])
cmd.sync_agent_server_reset(address)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
cleanup_controller(grpc_controller)
def test_inc_restore_failure_invalid_block(grpc_controller, grpc_replica1,
grpc_replica2,
grpc_controller_no_frontend,
grpc_fixed_dir_replica1,
grpc_fixed_dir_replica2,
backup_targets): # NOQA
# This case is for vfs backup only
for backup_target in backup_targets:
if "vfs" in backup_target:
break
assert backup_target
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
# start dr volume (no frontend)
dr_address = grpc_controller_no_frontend.address
start_no_frontend_volume(grpc_controller_no_frontend,
grpc_fixed_dir_replica1, grpc_fixed_dir_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(0, snap0_data, grpc_controller_no_frontend)
# mock inc restore error: invalid block
delta_file = "volume-delta-" + backup0_name + ".img"
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk + ".tmp".encode('utf-8')]
subprocess.check_output(command).strip()
cmd.backup_restore(dr_address, backup1)
# restore status should contain the error info
failed_restore, finished_restore = 0, 0
for i in range(RETRY_COUNTS):
failed_restore, finished_restore = 0, 0
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
failed_restore += 1
if not status["isRestoring"]:
finished_restore += 1
if failed_restore == 2 and finished_restore == 2:
break
time.sleep(RETRY_INTERVAL)
assert failed_restore == 2 and finished_restore == 2
assert path.exists(FIXED_REPLICA_PATH1 + delta_file)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file)
for blk in blocks:
command = ["mv", blk + ".tmp".encode('utf-8'), blk]
subprocess.check_output(command)
cleanup_no_frontend_volume(grpc_controller_no_frontend,
grpc_fixed_dir_replica1,
grpc_fixed_dir_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1])
cmd.sync_agent_server_reset(address)
cmd.sync_agent_server_reset(dr_address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)