def backup_test(
dev,
address, # NOQA
volume_name,
engine_name,
backup_target):
offset = 0
length = 128
snap1_data = random_string(length)
verify_data(dev, offset, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
backup1_info = create_backup(address, snap1, backup_target)
assert backup1_info["VolumeName"] == volume_name
assert backup1_info["Size"] == BLOCK_SIZE_STR
snap2_data = random_string(length)
verify_data(dev, offset, snap2_data)
snap2_checksum = checksum_dev(dev)
snap2 = cmd.snapshot_create(address)
backup2_info = create_backup(address, snap2, backup_target)
assert backup2_info["VolumeName"] == volume_name
assert backup2_info["Size"] == BLOCK_SIZE_STR
snap3_data = random_string(length)
verify_data(dev, offset, snap3_data)
snap3_checksum = checksum_dev(dev)
snap3 = cmd.snapshot_create(address)
backup3_info = create_backup(address, snap3, backup_target)
assert backup3_info["VolumeName"] == volume_name
assert backup3_info["Size"] == BLOCK_SIZE_STR
restore_with_frontend(address, engine_name, backup3_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap3_data
c = checksum_dev(dev)
assert c == snap3_checksum
rm_backups(address, engine_name, [backup3_info["URL"]])
restore_with_frontend(address, engine_name, backup1_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = checksum_dev(dev)
assert c == snap1_checksum
rm_backups(address, engine_name, [backup1_info["URL"]])
restore_with_frontend(address, engine_name, backup2_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap2_data
c = checksum_dev(dev)
assert c == snap2_checksum
rm_backups(address, engine_name, [backup2_info["URL"]])
def test_ha_remove_extra_disks(
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
address = grpc_controller.address
prepare_backup_dir(BACKUP_DIR)
open_replica(grpc_replica1)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
r1_url = grpc_replica1.url
v = grpc_controller.volume_start(replicas=[r1_url])
assert v.name == VOLUME_NAME
assert v.replicaCount == 1
replicas = grpc_controller.replica_list()
assert len(replicas) == 1
assert replicas[0].mode == "RW"
dev = get_blockdev(VOLUME_NAME)
wasted_data = random_string(128)
data_offset = 1024
verify_data(dev, data_offset, wasted_data)
# now replica1 contains extra data in a snapshot
cmd.snapshot_create(address)
cleanup_controller(grpc_controller)
open_replica(grpc_replica2)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
r2_url = grpc_replica2.url
v = grpc_controller.volume_start(replicas=[r2_url])
assert v.name == VOLUME_NAME
assert v.replicaCount == 1
replicas = grpc_controller.replica_list()
assert len(replicas) == 1
assert replicas[0].mode == "RW"
dev = get_blockdev(VOLUME_NAME)
data = random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
r1 = grpc_replica1.replica_reload()
print(r1)
cmd.add_replica(address, r1_url)
wait_for_rebuild_complete(address)
verify_data(dev, data_offset, data)
def backup_hole_with_backing_file_test(backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_backing_dev(grpc_backing_replica1,
grpc_backing_replica2,
grpc_backing_controller)
offset1 = 512
length1 = 256
offset2 = 640
length2 = 256
boundary_offset = 0
boundary_length = 4100 # just pass 4096 into next 4k
hole_offset = 2 * 1024 * 1024
hole_length = 1024
snap1_data = random_string(length1)
verify_data(dev, offset1, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
boundary_data_backup1 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup1 = read_dev(dev, hole_offset, hole_length)
backup1_info = create_backup(address, snap1, backup_target)
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
snap2_checksum = checksum_dev(dev)
snap2 = cmd.snapshot_create(address)
boundary_data_backup2 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup2 = read_dev(dev, hole_offset, hole_length)
backup2_info = create_backup(address, snap2, backup_target)
restore_with_frontend(address, ENGINE_BACKING_NAME,
backup1_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup1
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup1
c = checksum_dev(dev)
assert c == snap1_checksum
restore_with_frontend(address, ENGINE_BACKING_NAME,
backup2_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup2
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup2
c = checksum_dev(dev)
assert c == snap2_checksum
def test_backup_corrupt_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
address = grpc_controller.address
length = 128
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
# write two backup blocks
verify_data(dev, 0, random_string(length))
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup1 = create_backup(address, snap, backup_target)
# overwrite second backup block
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup2 = create_backup(address, snap, backup_target)
# check that the volume now has 3 blocks
# backup1 and backup2 share the first block
# and have different second blocks
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# corrupt backup1 config
cfg = findfile(BACKUP_DIR, "backup_" + backup1["Name"] + ".cfg")
corrupt_backup = open(cfg, "w")
assert corrupt_backup
assert corrupt_backup.write("{corrupt: definitely") > 0
corrupt_backup.close()
cmd.backup_rm(address, backup1["URL"])
# check that the volume now has 2 blocks
# backup2 still relies on the backup1 first block
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 2)
# remove backup 2 and check that all blocks are deleted
cmd.backup_rm(address, backup2["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 0)
# remove volume.cfg then delete the backup volume
cfg = findfile(finddir(BACKUP_DIR, VOLUME_NAME), "volume.cfg")
os.remove(cfg)
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME,
backup_target)[VOLUME_NAME]
assert "cannot find" in info["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_volume_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
offset = 0
length = 128
address = grpc_controller.address
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
snap_data = random_string(length)
verify_data(dev, offset, snap_data)
snap = cmd.snapshot_create(address)
backup_info = create_backup(address, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
url = get_backup_volume_url(backup_target, VOLUME_NAME)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect_volume(address, url)
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def __init__(self, dev, data, controller_addr):
self.dev = dev
self.data = data
self.controller_addr = controller_addr
self.data.write_and_verify_data(self.dev)
self.checksum = checksum_dev(self.dev)
self.name = cmd.snapshot_create(controller_addr)
def backup_with_backing_file_test(backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_backing_dev(grpc_backing_replica1,
grpc_backing_replica2,
grpc_backing_controller)
offset = 0
length = 256
snap0 = cmd.snapshot_create(address)
before = read_dev(dev, offset, length)
assert before != ""
snap0_checksum = checksum_dev(dev)
exists = read_from_backing_file(offset, length)
assert before == exists
backup0_info = create_backup(address, snap0, backup_target)
assert backup0_info["VolumeName"] == VOLUME_BACKING_NAME
backup_test(dev, address, VOLUME_BACKING_NAME,
ENGINE_BACKING_NAME, backup_target)
restore_with_frontend(address, ENGINE_BACKING_NAME,
backup0_info["URL"])
after = read_dev(dev, offset, length)
assert before == after
c = checksum_dev(dev)
assert c == snap0_checksum
rm_backups(address, ENGINE_BACKING_NAME, [backup0_info["URL"]])
def test_snapshot_revert_with_backing_file(grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
offset = 0
length = 256
snap0 = cmd.snapshot_create(address)
before = read_dev(dev, offset, length)
assert before != ""
info = cmd.snapshot_info(address)
assert len(info) == 2
assert VOLUME_HEAD in info
assert snap0 in info
exists = read_from_backing_file(offset, length)
assert before == exists
snapshot_revert_test(dev, address, ENGINE_BACKING_NAME)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
after = read_dev(dev, offset, length)
assert before == after
def test_backup_volume_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
offset = 0
length = 128
address = grpc_controller.address
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
snap_data = random_string(length)
verify_data(dev, offset, snap_data)
snap = cmd.snapshot_create(address)
backup_info = create_backup(address, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME, backup_target)
assert "cannot find" in info[VOLUME_NAME]["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_incremental_logic(grpc_replica1, grpc_replica2,
grpc_controller, backup_targets): # NOQA
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
address = grpc_controller.address
volume_name = VOLUME_NAME
engine_name = ENGINE_NAME
offset = 0
length = 128
# initial backup
snap1_data = random_string(length)
verify_data(dev, offset, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
backup1_info = create_backup(address, snap1, backup_target)
assert backup1_info["IsIncremental"] is False
# delta backup on top of initial backup
snap2_data = random_string(int(length / 2))
verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2_info = create_backup(address, snap2, backup_target)
assert backup2_info["IsIncremental"] is True
# delete the volume
cmd.sync_agent_server_reset(address)
grpc_controller = cleanup_controller(grpc_controller)
grpc_replica1 = cleanup_replica(grpc_replica1)
grpc_replica2 = cleanup_replica(grpc_replica2)
# recreate the volume
dev = get_dev(grpc_replica1,
grpc_replica2,
grpc_controller,
clean_backup_dir=False)
# empty initial backup after volume recreation
snap3 = cmd.snapshot_create(address)
backup3_info = create_backup(address, snap3, backup_target)
assert backup3_info["VolumeName"] == volume_name
assert backup3_info["Size"] == '0'
assert backup3_info["IsIncremental"] is False
# write half of snap1 onto head
snap4_data = snap1_data[:int(length / 2)]
assert len(snap4_data) == int(length / 2)
verify_data(dev, offset, snap4_data)
snap4_checksum = checksum_dev(dev)
assert snap4_checksum != snap1_checksum
snap4 = cmd.snapshot_create(address)
backup4_info = create_backup(address, snap4, backup_target)
assert backup4_info["IsIncremental"] is True
# restore initial backup
reset_volume(grpc_controller, grpc_replica1, grpc_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, engine_name, backup1_info["URL"])
assert read_dev(dev, offset, length) == snap1_data
assert checksum_dev(dev) == snap1_checksum
# restore final backup (half of snap1)
reset_volume(grpc_controller, grpc_replica1, grpc_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, engine_name, backup4_info["URL"])
assert checksum_dev(dev) == snap4_checksum
assert snap4_checksum != snap1_checksum
data = read_dev(dev, offset, length)
assert data[:int(length / 2)] == snap4_data
assert data[int(length / 2):] == '\x00' * int(length / 2)
rm_backups(address, engine_name, [
backup1_info["URL"], backup2_info["URL"], backup3_info["URL"],
backup4_info["URL"]
])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_S3_latest_unavailable(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
for backup_target in backup_targets:
if "s3://" not in backup_target:
continue
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
address = grpc_controller.address
volume_name = VOLUME_NAME
engine_name = ENGINE_NAME
offset = 0
length = 128
# initial backup
snap1_data = random_string(length)
verify_data(dev, offset, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
backup1_info = create_backup(address, snap1, backup_target)
# backup to be unavailable
snap2_data = random_string(length)
verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2_info = create_backup(address, snap2, backup_target)
# the gc after the restore will clean up the missing backup
cfg = findfile(BACKUP_DIR, "backup_" + backup2_info["Name"] + ".cfg")
os.remove(cfg)
# final full backup after unavailable backup
snap3_data = random_string(length)
verify_data(dev, offset, snap3_data)
snap3_checksum = checksum_dev(dev)
snap3 = cmd.snapshot_create(address)
backup3_info = create_backup(address, snap3, backup_target)
assert backup3_info["VolumeName"] == volume_name
assert backup3_info["Size"] == BLOCK_SIZE_STR
# write some stuff on head
head_data = random_string(length)
verify_data(dev, offset, head_data)
# test restore of the initial backup
reset_volume(grpc_controller, grpc_replica1, grpc_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, engine_name, backup1_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = checksum_dev(dev)
assert c == snap1_checksum
# test a restore for the final backup
reset_volume(grpc_controller, grpc_replica1, grpc_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, engine_name, backup3_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap3_data
c = checksum_dev(dev)
assert c == snap3_checksum
rm_backups(address, engine_name,
[backup1_info["URL"], backup3_info["URL"]])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def restore_to_file_with_backing_file_test(
backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
backing_dev = get_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
length0 = 4 * 1024
length1 = 256
length2 = 128
offset0 = 0
offset1 = length1 + offset0
offset2 = length2 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot.
# data in output image == data in backing
check_backing()
check_empty_volume(backing_dev)
snap0 = cmd.snapshot_create(address)
backup = create_backup(address, snap0, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset0, length0)
dev_checksum = checksum_dev(backing_dev)
assert volume_data != ""
assert volume_data == backing_data
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output0_raw = read_file(output_raw_path, offset0, length0)
output0_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE).encode('utf-8'))
assert output0_raw == backing_data
assert output0_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output0_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
output0_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output0_qcow2.decode('utf-8') == backing_data
assert output0_qcow2.decode('utf-8') == volume_data
assert output0_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
rm_backups(address, ENGINE_BACKING_NAME, [backup])
# create 1 snapshot with 256B data.
# output = snap1(offset0, length1) + backing(offset1, ...)
snap1_data = random_string(length1)
verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
backup = create_backup(address, snap1, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset1, length0 - offset1)
dev_checksum = checksum_dev(backing_dev)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output1_raw_snap1 = read_file(output_raw_path, offset0, length1)
output1_raw_backing = read_file(output_raw_path, offset1,
length0 - offset1)
output1_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE).encode('utf-8'))
assert output1_raw_snap1 == snap1_data
assert output1_raw_backing == backing_data
assert output1_raw_snap1 + output1_raw_backing == volume_data
assert output1_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output1_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output1_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output1_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output1_qcow2_snap1.decode('utf-8') == snap1_data
assert output1_qcow2_backing.decode('utf-8') == backing_data
assert output1_qcow2_snap1.decode('utf-8') + \
output1_qcow2_backing.decode('utf-8') == volume_data
assert output1_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
rm_snaps(address, [snap1])
rm_backups(address, ENGINE_BACKING_NAME, [backup])
check_backing()
check_empty_volume(backing_dev)
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2) + backing(offset2, ...)
snap1_data = random_string(length1)
verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
snap2_data = random_string(length2)
verify_data(backing_dev, offset0, snap2_data)
snap2 = cmd.snapshot_create(address)
backup = create_backup(address, snap2, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset1, length0 - offset1)
dev_checksum = checksum_dev(backing_dev)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(output_raw_path, offset0, length2)
output2_raw_snap1 = read_file(output_raw_path, offset2, length1 - length2)
output2_raw_backing = read_file(output_raw_path, offset1,
length0 - offset1)
output2_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE).encode('utf-8'))
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset2:length1]
assert output2_raw_backing == backing_data
assert \
volume_data == \
output2_raw_snap2 + output2_raw_snap1 + output2_raw_backing
assert output2_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length2)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset2, length1 - length2)
output2_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output2_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output2_qcow2_snap2.decode('utf-8') == snap2_data
assert output2_qcow2_snap1.decode('utf-8') == snap1_data[offset2:length1]
assert output2_qcow2_backing.decode('utf-8') == backing_data
assert \
volume_data == \
output2_qcow2_snap2.decode('utf-8') + \
output2_qcow2_snap1.decode('utf-8') + \
output1_qcow2_backing.decode('utf-8')
assert output2_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
rm_snaps(address, [snap1, snap2])
rm_backups(address, ENGINE_BACKING_NAME, [backup])
check_backing()
check_empty_volume(backing_dev)
def test_backup_type(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
for backup_target in backup_targets:
address = grpc_controller.address
block_size = 2 * 1024 * 1024
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, block_size, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)
backup0_url = backup0["URL"]
assert backup0['IsIncremental'] is False
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)
backup1_url = backup1["URL"]
assert backup1['IsIncremental'] is True
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, block_size, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)
backup2_url = backup2["URL"]
assert backup2['IsIncremental'] is True
rm_backups(address, ENGINE_NAME, [backup2_url])
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, block_size, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)
backup3_url = backup3["URL"]
assert backup3['IsIncremental'] is True
# full backup: backup the same snapshot twice
backup3 = create_backup(address, snap3, backup_target)
backup3_url = backup3["URL"]
assert backup3['IsIncremental'] is False
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)
backup4_url = backup4["URL"]
assert backup4['IsIncremental'] is True
rm_backups(address, ENGINE_NAME,
[backup0_url, backup1_url, backup3_url, backup4_url])
cmd.sync_agent_server_reset(address)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
cleanup_controller(grpc_controller)
def test_backup_volume_list(
grpc_replica_client,
grpc_controller_client, # NOQA
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
"""
Test backup volume list
Context:
We want to make sure that an error when listing a single backup volume
does not stop us from listing all the other backup volumes. Otherwise a
single faulty backup can block the retrieval of all known backup volumes.
Steps:
1. Create a volume(1,2) and attach to the current node
2. write some data to volume(1,2)
3. Create a backup(1) of volume(1,2)
4. request a backup list
5. verify backup list contains no error messages for volume(1,2)
6. verify backup list contains backup(1) for volume(1,2)
7. place a file named "*****@*****.**"
into the backups folder of volume(1)
8. request a backup list
9. verify backup list contains no error messages for volume(1,2)
10. verify backup list contains backup(1) for volume(1,2)
11. delete backup volumes(1 & 2)
12. cleanup
"""
# create a second volume
grpc2_replica1 = grpc_replica_client(REPLICA_2_NAME + "-1")
grpc2_replica2 = grpc_replica_client(REPLICA_2_NAME + "-2")
grpc2_controller = grpc_controller_client(ENGINE2_NAME, VOLUME2_NAME)
offset = 0
length = 128
address = grpc_controller.address
address2 = grpc2_controller.address
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
dev2 = get_dev(grpc2_replica1, grpc2_replica2, grpc2_controller)
# create a regular backup
snap_data = random_string(length)
verify_data(dev, offset, snap_data)
snap = cmd.snapshot_create(address)
backup_info = create_backup(address, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
# create a regular backup on volume 2
verify_data(dev2, offset, random_string(length))
snap = cmd.snapshot_create(address2)
backup_info = create_backup(address2, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME2_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
# request a volume list
info = cmd.backup_volume_list(address,
"",
backup_target,
include_backup_details=True)
assert info[VOLUME_NAME]["Name"] == VOLUME_NAME
assert len(info[VOLUME_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME_NAME]["Messages"]
assert info[VOLUME2_NAME]["Name"] == VOLUME2_NAME
assert len(info[VOLUME2_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME2_NAME]["Messages"]
# place badly named backup.cfg file
# we want the list call to return all valid files correctly
backup_dir = os.path.join(finddir(BACKUP_DIR, VOLUME_NAME), "backups")
cfg = open(os.path.join(backup_dir, "*****@*****.**"), "w")
cfg.close()
info = cmd.backup_volume_list(address,
"",
backup_target,
include_backup_details=True)
assert info[VOLUME_NAME]["Name"] == VOLUME_NAME
assert len(info[VOLUME_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME_NAME]["Messages"]
assert info[VOLUME2_NAME]["Name"] == VOLUME2_NAME
assert len(info[VOLUME2_NAME]["Backups"]) == 1
assert MESSAGE_TYPE_ERROR not in info[VOLUME2_NAME]["Messages"]
# remove the volume with the badly named backup.cfg
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address,
VOLUME_NAME,
backup_target,
include_backup_details=True)
assert "cannot find" in info[VOLUME_NAME]["Messages"]["error"]
# remove volume 2 backups
cmd.backup_volume_rm(address, VOLUME2_NAME, backup_target)
info = cmd.backup_volume_list(address,
VOLUME2_NAME,
backup_target,
include_backup_details=True)
assert "cannot find" in info[VOLUME2_NAME]["Messages"]["error"]
# cleanup volume 1
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
# cleanup volume 2
cmd.sync_agent_server_reset(address2)
cleanup_controller(grpc2_controller)
cleanup_replica(grpc2_replica1)
cleanup_replica(grpc2_replica2)
def snapshot_mounted_filesystem_test(volume_name, dev, address,
engine_name): # NOQA
dev_path = dev.dev
mnt_path = "/tmp/mnt-" + volume_name
test_file = mnt_path + "/test"
length = 128
print("dev_path: " + dev_path + "\n")
print("mnt_path: " + mnt_path + "\n")
# create & mount a ext4 filesystem on dev
nsenter_cmd = [
"nsenter", "--mount=/host/proc/1/ns/mnt", "--net=/host/proc/1/ns/net",
"--"
]
mount_cmd = nsenter_cmd + ["mount", "--make-shared", dev_path, mnt_path]
umount_cmd = nsenter_cmd + ["umount", mnt_path]
findmnt_cmd = nsenter_cmd + ["findmnt", dev_path]
subprocess.check_call(nsenter_cmd + ["mkfs.ext4", dev_path])
subprocess.check_call(nsenter_cmd + ["mkdir", "-p", mnt_path])
subprocess.check_call(mount_cmd)
subprocess.check_call(findmnt_cmd)
def checksum_test_file():
read_cmd = nsenter_cmd + ["cat", test_file]
data = subprocess.check_output(read_cmd)
return checksum_data(str(data).encode('utf-8'))
def write_test_file():
# beware don't touch this write command
data = random_string(length)
write_cmd = [
"/bin/sh -c '/bin/echo", '"' + data + '"', ">", test_file + "'"
]
shell_cmd = " ".join(nsenter_cmd + write_cmd)
subprocess.check_call(shell_cmd, shell=True)
return checksum_test_file()
# create snapshot1 with empty fs
# NOTE: we cannot use checksum_dev since it assumes
# asci data for device data instead of raw bytes
snap1 = cmd.snapshot_create(address)
# create snapshot2 with a new test file
test2_checksum = write_test_file()
snap2 = cmd.snapshot_create(address)
# create snapshot3 overwriting the test file
test3_checksum = write_test_file()
snap3 = cmd.snapshot_create(address)
# verify existence of the snapshots
snapshots = cmd.snapshot_ls(address)
assert snap1 in snapshots
assert snap2 in snapshots
assert snap3 in snapshots
# unmount the volume, since each revert will shutdown the device
subprocess.check_call(umount_cmd)
# restore snapshots 1,2,3 & verify filesystem state
print("\nsnapshot_revert_with_frontend snap1 begin")
snapshot_revert_with_frontend(address, engine_name, snap1)
print("snapshot_revert_with_frontend snap1 finish\n")
subprocess.check_call(mount_cmd)
# should error since the file does not exist in snapshot 1
with pytest.raises(subprocess.CalledProcessError):
print("is expected error, since the file does not exist.")
checksum_test_file()
subprocess.check_call(umount_cmd)
print("\nsnapshot_revert_with_frontend snap2 begin")
snapshot_revert_with_frontend(address, engine_name, snap2)
print("snapshot_revert_with_frontend snap2 finish\n")
subprocess.check_call(mount_cmd)
assert checksum_test_file() == test2_checksum
subprocess.check_call(umount_cmd)
print("\nsnapshot_revert_with_frontend snap3 begin")
snapshot_revert_with_frontend(address, engine_name, snap3)
print("snapshot_revert_with_frontend snap3 finish\n")
subprocess.check_call(mount_cmd)
assert checksum_test_file() == test3_checksum
subprocess.check_call(umount_cmd)
# remove the created mount folder
subprocess.check_call(nsenter_cmd + ["rmdir", mnt_path])
def test_inc_restore_failure_invalid_block(grpc_controller, grpc_replica1,
grpc_replica2,
grpc_controller_no_frontend,
grpc_fixed_dir_replica1,
grpc_fixed_dir_replica2,
backup_targets): # NOQA
# This case is for vfs backup only
for backup_target in backup_targets:
if "vfs" in backup_target:
break
assert backup_target
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
# start dr volume (no frontend)
dr_address = grpc_controller_no_frontend.address
start_no_frontend_volume(grpc_controller_no_frontend,
grpc_fixed_dir_replica1, grpc_fixed_dir_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(0, snap0_data, grpc_controller_no_frontend)
# mock inc restore error: invalid block
delta_file = "volume-delta-" + backup0_name + ".img"
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk + ".tmp".encode('utf-8')]
subprocess.check_output(command).strip()
cmd.backup_restore(dr_address, backup1)
# restore status should contain the error info
failed_restore, finished_restore = 0, 0
for i in range(RETRY_COUNTS):
failed_restore, finished_restore = 0, 0
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
failed_restore += 1
if not status["isRestoring"]:
finished_restore += 1
if failed_restore == 2 and finished_restore == 2:
break
time.sleep(RETRY_INTERVAL)
assert failed_restore == 2 and finished_restore == 2
assert path.exists(FIXED_REPLICA_PATH1 + delta_file)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file)
for blk in blocks:
command = ["mv", blk + ".tmp".encode('utf-8'), blk]
subprocess.check_output(command)
cleanup_no_frontend_volume(grpc_controller_no_frontend,
grpc_fixed_dir_replica1,
grpc_fixed_dir_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1])
cmd.sync_agent_server_reset(address)
cmd.sync_agent_server_reset(dr_address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def inc_restore_failure_cleanup_error_test(grpc_controller, grpc_replica1,
grpc_replica2, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2,
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(0, snap0_data, grpc_dr_controller)
# mock inc restore crash/error: cannot clean up the delta file path
delta_file = "volume-delta-" + backup0_name + ".img"
command = ["mkdir", "-p", FIXED_REPLICA_PATH1 + delta_file + "/dir"]
subprocess.check_output(command).strip()
command = ["mkdir", "-p", FIXED_REPLICA_PATH2 + delta_file + "/dir"]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup1)
assert "failed to clean up the existing file" in e.value.stdout
command = ["rm", "-r", FIXED_REPLICA_PATH1 + delta_file]
subprocess.check_output(command).strip()
command = ["rm", "-r", FIXED_REPLICA_PATH2 + delta_file]
subprocess.check_output(command).strip()
# the restore status will be reverted/keep unchanged
# if an error is triggered before the actual restore is performed
rs = cmd.restore_status(dr_address)
for status in rs.values():
assert not status["isRestoring"]
assert status['backupURL'] == backup0
assert 'error' not in status
assert status['progress'] == 100
assert status['state'] == "complete"
cleanup_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1])
cmd.sync_agent_server_reset(address)
cmd.sync_agent_server_reset(dr_address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def restore_inc_test(grpc_controller, grpc_replica1, grpc_replica2,
grpc_dr_controller, grpc_dr_replica1, grpc_dr_replica2,
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
# mock restore crash/error:
# By adding attribute `immutable`, Longhorn cannot create a file
# for the restore. Then the following restore command will fail.
command = ["chattr", "+i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "+i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup0)
assert "operation not permitted" in e.value.stdout
# the restore status will be reverted/keep unchanged
# if an error is triggered before the actual restore is performed
rs = cmd.restore_status(dr_address)
for status in rs.values():
assert not status["isRestoring"]
assert not status['backupURL']
assert 'error' not in status
assert 'progress' not in status
assert not status['state']
command = ["chattr", "-i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "-i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
cmd.sync_agent_server_reset(dr_address)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(0, snap0_data, grpc_dr_controller)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
cmd.backup_restore(dr_address, backup1)
wait_for_restore_completion(dr_address, backup1)
verify_no_frontend_data(0, data1, grpc_dr_controller)
delta_file1 = "volume-delta-" + backup0_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
cmd.backup_restore(dr_address, backup2)
wait_for_restore_completion(dr_address, backup2)
verify_no_frontend_data(0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition: duplicate inc restore calls
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup2)
wait_for_restore_completion(dr_address, backup2)
assert "already restored backup" in e.value.stdout
data3 = zero_string * length3 + data2[length3:length2]
cmd.backup_restore(dr_address, backup3)
wait_for_restore_completion(dr_address, backup3)
verify_no_frontend_data(0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# This inc restore will fall back to full restore
cmd.backup_restore(dr_address, backup4)
wait_for_restore_completion(dr_address, backup4)
verify_no_frontend_data(0, snap4_data, grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
# check if the tmp files during this special full restore are cleaned up.
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*snap_tmp"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cmd.sync_agent_server_reset(dr_address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def restore_inc_test(
grpc_engine_manager, # NOQA
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2, # NOQA
grpc_dr_controller, # NOQA
grpc_dr_replica1,
grpc_dr_replica2, # NOQA
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
# mock restore crash/error:
# By adding attribute `immutable`, Longhorn cannot create a file
# for the restore. Then the following restore command will fail.
command = ["chattr", "+i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "+i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup0)
assert "operation not permitted" in e.value.stdout
command = ["chattr", "-i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "-i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(grpc_engine_manager, 0, snap0_data,
grpc_dr_controller)
delta_file1 = "volume-delta-" + backup0_name + ".img"
# mock inc restore crash/error: cannot clean up the delta file path
command = ["mkdir", "-p", FIXED_REPLICA_PATH1 + delta_file1 + "/dir"]
subprocess.check_output(command).strip()
command = ["mkdir", "-p", FIXED_REPLICA_PATH2 + delta_file1 + "/dir"]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError):
cmd.restore_inc(dr_address, backup1, backup0_name)
command = ["rm", "-r", FIXED_REPLICA_PATH1 + delta_file1]
subprocess.check_output(command).strip()
command = ["rm", "-r", FIXED_REPLICA_PATH2 + delta_file1]
subprocess.check_output(command).strip()
if "vfs" in backup_target:
# restore status should contain the error info
failed_restore, finished_restore = 0, 0
for i in range(RETRY_COUNTS):
failed_restore, finished_restore = 0, 0
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'failed to clean up the existing file' in\
status['error']
failed_restore += 1
if not status["isRestoring"]:
finished_restore += 1
if failed_restore == 2 and finished_restore == 2:
break
time.sleep(RETRY_INTERVAL)
assert failed_restore == 2 and finished_restore == 2
# mock inc restore error: invalid block
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk + ".tmp".encode('utf-8')]
subprocess.check_output(command).strip()
cmd.restore_inc(dr_address, backup1, backup0_name)
# restore status should contain the error info
failed_restore, finished_restore = 0, 0
for i in range(RETRY_COUNTS):
failed_restore, finished_restore = 0, 0
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
failed_restore += 1
if not status["isRestoring"]:
finished_restore += 1
if failed_restore == 2 and finished_restore == 2:
break
time.sleep(RETRY_INTERVAL)
assert failed_restore == 2 and finished_restore == 2
assert path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file1)
for blk in blocks:
command = ["mv", blk + ".tmp".encode('utf-8'), blk]
subprocess.check_output(command)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
restore_incrementally(dr_address, backup1, backup0_name)
verify_no_frontend_data(grpc_engine_manager, 0, data1, grpc_dr_controller)
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
restore_incrementally(dr_address, backup2, backup1_name)
verify_no_frontend_data(grpc_engine_manager, 0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition
with pytest.raises(subprocess.CalledProcessError) as e:
restore_incrementally(dr_address, backup1, backup0_name)
assert "doesn't match field LastRestored" in e.value.stdout
data3 = zero_string * length3 + data2[length3:length2]
restore_incrementally(dr_address, backup3, backup2_name)
verify_no_frontend_data(grpc_engine_manager, 0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# actually it is full restoration
restore_incrementally(dr_address, backup4, backup3_name)
verify_no_frontend_data(grpc_engine_manager, 0, snap4_data,
grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*tempoary"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_block_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
address = grpc_controller.address
length = 128
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
# write two backup block
verify_data(dev, 0, random_string(length))
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup1 = create_backup(address, snap, backup_target)
assert backup1["VolumeName"] == VOLUME_NAME
assert backup1["Size"] == str(BLOCK_SIZE * 2)
assert snap in backup1["SnapshotName"]
# test block deduplication
backup1_duplicate = create_backup(address, snap, backup_target)
assert backup1_duplicate["VolumeName"] == VOLUME_NAME
assert backup1_duplicate["Size"] == str(BLOCK_SIZE * 2)
assert snap in backup1_duplicate["SnapshotName"]
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 2)
# overwrite second backup block
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup2 = create_backup(address, snap, backup_target)
assert backup2["VolumeName"] == VOLUME_NAME
assert backup2["Size"] == str(BLOCK_SIZE * 2)
assert snap in backup2["SnapshotName"]
# check that the volume now has 3 blocks
# backup1 and backup2 share the first block
# and have different second blocks
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# remove backup 1 duplicate
# this should not change the blocks on disk
# since all blocks are still required
cmd.backup_rm(address, backup1_duplicate["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# remove backup 1
# the volume should now have 2 blocks
# blk1 from backup1 should still be present
# since it's required by backup 2
cmd.backup_rm(address, backup1["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 2)
# remove the last remaining backup 2
# this should remove all blocks
cmd.backup_rm(address, backup2["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 0)
# cleanup the backup volume
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME,
backup_target)[VOLUME_NAME]
assert "cannot find" in info["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_backup_block_no_cleanup(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
address = grpc_controller.address
length = 128
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
# write two backup blocks
verify_data(dev, 0, random_string(length))
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup1 = create_backup(address, snap, backup_target)
assert backup1["VolumeName"] == VOLUME_NAME
assert backup1["Size"] == str(BLOCK_SIZE * 2)
assert snap in backup1["SnapshotName"]
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 2)
# overwrite second backup block
verify_data(dev, BLOCK_SIZE, random_string(length))
snap = cmd.snapshot_create(address)
backup2 = create_backup(address, snap, backup_target)
assert backup2["VolumeName"] == VOLUME_NAME
assert backup2["Size"] == str(BLOCK_SIZE * 2)
assert snap in backup2["SnapshotName"]
# check that the volume now has 3 blocks
# backup1 and backup2 share the first block
# and have different second blocks
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# create an artificial in progress backup
# that will stop the gc from removing blocks
in_progress_backup_file = create_in_progress_backup_file(VOLUME_NAME)
# remove backup 1 the volume should still have 3 blocks
cmd.backup_rm(address, backup1["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 3)
# remove the in progress backup
os.remove(in_progress_backup_file)
# remove the last remaining backup 2
# this should remove all blocks
# including the orphaned block from backup 1
cmd.backup_rm(address, backup2["URL"])
check_backup_volume_block_count(address, VOLUME_NAME, backup_target, 0)
# cleanup the backup volume
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME,
backup_target)[VOLUME_NAME]
assert "cannot find" in info["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def snapshot_tree_create_node(dev, address, offset, length, snap, data, name):
data[name] = random_string(length)
verify_data(dev, offset, data[name])
snap[name] = cmd.snapshot_create(address)
def backup_hole_with_backing_file_test(
backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
volume_name = grpc_backing_controller.volume_get().name
assert volume_name == VOLUME_BACKING_NAME
offset1 = 512
length1 = 256
offset2 = 640
length2 = 256
boundary_offset = 0
boundary_length = 4100 # just pass 4096 into next 4k
hole_offset = 2 * 1024 * 1024
hole_length = 1024
snap1_data = random_string(length1)
verify_data(dev, offset1, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
boundary_data_backup1 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup1 = read_dev(dev, hole_offset, hole_length)
backup1_info = create_backup(address,
snap1,
backup_target,
backing_image_name=BACKING_IMAGE_NAME,
backing_image_url=BACKING_IMAGE_URL)
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
snap2_checksum = checksum_dev(dev)
snap2 = cmd.snapshot_create(address)
boundary_data_backup2 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup2 = read_dev(dev, hole_offset, hole_length)
backup2_info = create_backup(address,
snap2,
backup_target,
backing_image_name=BACKING_IMAGE_NAME,
backing_image_url=BACKING_IMAGE_URL)
assert backup2_info["VolumeBackingImageName"] == BACKING_IMAGE_NAME
assert backup2_info["VolumeBackingImageURL"] == BACKING_IMAGE_URL
reset_volume(grpc_backing_controller, grpc_backing_replica1,
grpc_backing_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, ENGINE_BACKING_NAME, backup1_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup1
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup1
c = checksum_dev(dev)
assert c == snap1_checksum
reset_volume(grpc_backing_controller, grpc_backing_replica1,
grpc_backing_replica2)
dev = get_blockdev(volume_name)
restore_with_frontend(address, ENGINE_BACKING_NAME, backup2_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup2
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup2
c = checksum_dev(dev)
assert c == snap2_checksum
def test_ha_single_replica_rebuild(
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
address = grpc_controller.address
open_replica(grpc_replica1)
open_replica(grpc_replica2)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
r1_url = grpc_replica1.url
r2_url = grpc_replica2.url
v = grpc_controller.volume_start(replicas=[r1_url, r2_url])
assert v.replicaCount == 2
replicas = grpc_controller.replica_list()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev(VOLUME_NAME)
data = random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
# Cleanup replica2
cleanup_replica(grpc_replica2)
verify_async(dev, 10, 128, 1)
verify_replica_state(grpc_controller, r2_url, "ERR")
verify_read(dev, data_offset, data)
grpc_controller.replica_delete(replicas[1].address)
# Rebuild replica2
open_replica(grpc_replica2)
cmd.add_replica(address, r2_url)
wait_for_rebuild_complete(address)
verify_async(dev, 10, 128, 1)
verify_replica_state(grpc_controller, r2_url, "RW")
verify_read(dev, data_offset, data)
# WORKAROUND for unable to remove the parent of volume head
newsnap = cmd.snapshot_create(address)
info = cmd.snapshot_info(address)
assert len(info) == 3
sysnap = info[newsnap]["parent"]
assert info[sysnap]["parent"] == ""
assert newsnap in info[sysnap]["children"]
assert info[sysnap]["usercreated"] is False
assert info[sysnap]["removed"] is False
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
info = cmd.snapshot_info(address)
assert len(info) == 2
assert info[newsnap] is not None
assert info[VOLUME_HEAD] is not None
def test_backup_lock(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
"""
Test backup locks
Context:
The idea is to implement a locking mechanism that utilizes the backupstore,
to prevent the following dangerous cases of concurrent operations.
- prevent backup deletion during backup restoration
- prevent backup deletion while a backup is in progress
- prevent backup creation during backup deletion
- prevent backup restoration during backup deletion
Steps:
1. Create a volume(1) and attach to the current node
2. create a backup(1) of volume(1)
3. verify backup(1) creation completed
4. write some data to volume(1)
5. create an active lock of type Delete
6. create a backup(2) of volume(1)
7. verify backup(2) creation timed out
8. delete active lock of type Delete
9. create an active lock of type Delete
10. restore backup(1)
11. verify backup(1) restore timed out
12. delete active lock of type Delete
13. restore backup(1)
14. verify backup(1) restore completed
15. create an active lock of type Restore
16. delete backup(1)
17. verify backup(1) deletion timed out
18. delete active lock of type Restore
19. delete backup(1)
20. verify backup(1) deletion completed
21. cleanup
"""
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
# create a regular backup
address = grpc_controller.address
offset = 0
length = 128
snap1_data = random_string(length)
verify_data(dev, offset, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
# create a backup to create the volume
info = create_backup(address, snap1, backup_target)
assert info["VolumeName"] == VOLUME_NAME
assert info["Size"] == BLOCK_SIZE_STR
assert snap1 in info["SnapshotName"]
# backup should error out with timeout
# because of delete lock
create_delete_lock(True)
with pytest.raises(subprocess.CalledProcessError):
create_backup(address, snap1, backup_target)
remove_lock_file(DELETE_LOCK)
# restore should error out with timeout
# because of delete lock
create_delete_lock(True)
with pytest.raises(subprocess.CalledProcessError):
restore_with_frontend(address, ENGINE_NAME, info["URL"])
remove_lock_file(DELETE_LOCK)
# restore should succeed now, that there is no active delete lock
restore_with_frontend(address, ENGINE_NAME, info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = checksum_dev(dev)
assert c == snap1_checksum
# delete should error out with timeout
# because of restore lock
create_restore_lock(True)
with pytest.raises(subprocess.CalledProcessError):
rm_backups(address, ENGINE_NAME, [info["URL"]])
remove_lock_file(RESTORE_LOCK)
# delete should succeed now, that there is no active restore lock
rm_backups(address, ENGINE_NAME, [info["URL"]])
# cleanup volume 1
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def restore_inc_test(grpc_engine_manager, # NOQA
grpc_controller, # NOQA
grpc_replica1, grpc_replica2, # NOQA
grpc_dr_controller, # NOQA
grpc_dr_replica1, grpc_dr_replica2, # NOQA
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_engine_manager,
grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(grpc_engine_manager,
0, snap0_data, grpc_dr_controller)
# mock restore crash/error
delta_file1 = "volume-delta-" + backup0_name + ".img"
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk+".tmp".encode('utf-8')]
subprocess.check_output(command).strip()
# should fail
is_failed = False
cmd.restore_inc(dr_address, backup1, backup0_name)
for i in range(RETRY_COUNTS):
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
is_failed = True
if is_failed:
break
time.sleep(RETRY_INTERVAL)
assert is_failed
assert path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file1)
for blk in blocks:
command = ["mv", blk+".tmp".encode('utf-8'), blk]
subprocess.check_output(command)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
# race condition: last restoration has failed
# but `isRestoring` hasn't been cleanup
for i in range(RETRY_COUNTS):
try:
restore_incrementally(dr_address, backup1, backup0_name)
break
except subprocess.CalledProcessError as e:
if "already in progress" not in e.output:
time.sleep(RETRY_INTERVAL)
else:
raise e
verify_no_frontend_data(grpc_engine_manager,
0, data1, grpc_dr_controller)
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
restore_incrementally(dr_address, backup2, backup1_name)
verify_no_frontend_data(grpc_engine_manager,
0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition
with pytest.raises(subprocess.CalledProcessError) as e:
restore_incrementally(dr_address, backup1, backup0_name)
assert "doesn't match lastRestored" in e
data3 = zero_string * length3 + data2[length3:length2]
restore_incrementally(dr_address, backup3, backup2_name)
verify_no_frontend_data(grpc_engine_manager,
0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# actually it is full restoration
restore_incrementally(dr_address, backup4, backup3_name)
verify_no_frontend_data(grpc_engine_manager,
0, snap4_data, grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*tempoary"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_engine_manager,
grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def backup_core(bin, engine_manager_client, # NOQA
grpc_controller_client, # NOQA
grpc_replica_client, # NOQA
grpc_replica_client2, # NOQA
backup_target):
open_replica(grpc_replica_client)
open_replica(grpc_replica_client2)
r1_url = grpc_replica_client.url
r2_url = grpc_replica_client2.url
v = grpc_controller_client.volume_start(replicas=[
r1_url,
r2_url,
])
assert v.replicaCount == 2
env = dict(os.environ)
backup_type = urlparse(backup_target).scheme
# create & process backup1
snapshot1 = snapshot_create(grpc_controller_client.address)
output = grpc_replica_client.replica_get().chain[1]
assert output == 'volume-snap-{}.img'.format(snapshot1)
backup1 = backup_create(grpc_controller_client.address,
snapshot1, backup_target,
{'name': 'backup1', 'type': backup_type})
backup1_info = backup_inspect(grpc_controller_client.address, backup1)
assert backup1_info["URL"] == backup1
assert backup1_info["IsIncremental"] is False
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["VolumeSize"] == SIZE_STR
assert backup1_info["SnapshotName"] == snapshot1
assert len(backup1_info["Labels"]) == 2
assert backup1_info["Labels"]["name"] == "backup1"
assert backup1_info["Labels"]["type"] == backup_type
# create & process backup2
snapshot2 = snapshot_create(grpc_controller_client.address)
output = grpc_replica_client.replica_get().chain[1]
assert output == 'volume-snap-{}.img'.format(snapshot2)
backup2 = backup_create(grpc_controller_client.address,
snapshot2, backup_target)
backup2_info = backup_inspect(grpc_controller_client.address, backup2)
assert backup2_info["URL"] == backup2
assert backup2_info["IsIncremental"] is True
assert backup2_info["VolumeName"] == VOLUME_NAME
assert backup2_info["VolumeSize"] == SIZE_STR
assert backup2_info["SnapshotName"] == snapshot2
if backup2_info["Labels"] is not None:
assert len(backup2_info["Labels"]) == 0
# list all known backups for volume
volume_info = backup_volume_list(grpc_controller_client.address,
VOLUME_NAME, backup_target,
include_backup_details=True)[VOLUME_NAME]
assert volume_info["Name"] == VOLUME_NAME
assert volume_info["Size"] == SIZE_STR
backup_list = volume_info["Backups"]
assert backup_list[backup1]["URL"] == backup1_info["URL"]
assert backup_list[backup1]["SnapshotName"] == backup1_info["SnapshotName"]
assert backup_list[backup1]["Size"] == backup1_info["Size"]
assert backup_list[backup1]["Created"] == backup1_info["Created"]
assert backup_list[backup1]["Messages"] is None
assert backup_list[backup2]["URL"] == backup2_info["URL"]
assert backup_list[backup2]["SnapshotName"] == backup2_info["SnapshotName"]
assert backup_list[backup2]["Size"] == backup2_info["Size"]
assert backup_list[backup2]["Created"] == backup2_info["Created"]
assert backup_list[backup2]["Messages"] is None
# test that corrupt backups are signaled during a list operation
# https://github.com/longhorn/longhorn/issues/1212
volume_dir = finddir(BACKUP_DIR, VOLUME_NAME)
assert volume_dir
assert os.path.exists(volume_dir)
backup_dir = os.path.join(volume_dir, "backups")
assert os.path.exists(backup_dir)
backup_cfg_name = "backup_" + backup2_info["Name"] + ".cfg"
assert backup_cfg_name
backup_cfg_path = findfile(backup_dir, backup_cfg_name)
assert os.path.exists(backup_cfg_path)
backup_tmp_cfg_path = os.path.join(volume_dir, backup_cfg_name)
os.rename(backup_cfg_path, backup_tmp_cfg_path)
assert os.path.exists(backup_tmp_cfg_path)
corrupt_backup = open(backup_cfg_path, "w")
assert corrupt_backup
assert corrupt_backup.write("{corrupt: definitely") > 0
corrupt_backup.close()
# request the new backup list
volume_info = backup_volume_list(grpc_controller_client.address,
VOLUME_NAME, backup_target,
include_backup_details=True)[VOLUME_NAME]
assert volume_info["Name"] == VOLUME_NAME
backup_list = volume_info["Backups"]
assert backup_list[backup1]["URL"] == backup1_info["URL"]
assert backup_list[backup1]["Messages"] is None
assert backup_list[backup2]["URL"] == backup2_info["URL"]
assert MESSAGE_TYPE_ERROR in backup_list[backup2]["Messages"]
# we still want to fail inspects, since they operate on urls
# with no guarantee of backup existence
with pytest.raises(subprocess.CalledProcessError):
backup_inspect(grpc_controller_client.address, backup2)
# switch back to valid cfg
os.rename(backup_tmp_cfg_path, backup_cfg_path)
assert backup_inspect(grpc_controller_client.address, backup2)
# test that list returns a volume_info with an error message
# for a missing volume.cfg instead of failing with an error
# https://github.com/rancher/longhorn/issues/399
volume_cfg_path = findfile(volume_dir, VOLUME_CONFIG_FILE)
assert os.path.exists(volume_cfg_path)
volume_tmp_cfg_path = volume_cfg_path.replace(
VOLUME_CONFIG_FILE, VOLUME_TMP_CONFIG_FILE)
os.rename(volume_cfg_path, volume_tmp_cfg_path)
assert os.path.exists(volume_tmp_cfg_path)
volume_info = backup_volume_list(grpc_controller_client.address,
"", backup_target)
assert MESSAGE_TYPE_ERROR in volume_info[VOLUME_NAME]["Messages"]
os.rename(volume_tmp_cfg_path, volume_cfg_path)
assert os.path.exists(volume_cfg_path)
volume_info = backup_volume_list(grpc_controller_client.address,
"", backup_target)
assert volume_info[VOLUME_NAME]["Messages"] is not None
assert MESSAGE_TYPE_ERROR not in volume_info[VOLUME_NAME]["Messages"]
# backup doesn't exists so it should error
with pytest.raises(subprocess.CalledProcessError):
url = backup_target + "?backup=backup-unk" + "&volume=" + VOLUME_NAME
backup_inspect(grpc_controller_client.address, url)
# this returns unsupported driver since `bad` is not a known scheme
with pytest.raises(subprocess.CalledProcessError):
backup_inspect(grpc_controller_client.address, "bad://xxx")
restore_backup(engine_manager_client,
bin, grpc_controller_client.address,
backup1, env, grpc_controller_client)
restore_backup(engine_manager_client,
bin, grpc_controller_client.address,
backup2, env, grpc_controller_client)
# remove backups + volume
backup_rm(grpc_controller_client.address, backup1)
backup_rm(grpc_controller_client.address, backup2)
backup_volume_rm(grpc_controller_client.address,
VOLUME_NAME, backup_target)
assert os.path.exists(BACKUP_DIR)
assert not os.path.exists(volume_cfg_path)
grpc_controller_client.volume_frontend_start(
frontend=FRONTEND_TGT_BLOCKDEV)
v = grpc_controller_client.volume_get()
assert v.frontendState == "up"
def restore_to_file_without_backing_file_test(
backup_target, # NOQA
grpc_controller, # NOQA
grpc_replica1, # NOQA
grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
length0 = 256
length1 = 128
offset0 = 0
offset1 = length1 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot for converting to init state.
snap0 = cmd.snapshot_create(address)
# create 1 snapshot with 256B data.
# output = snap2(offset0, length1)
snap1_data = random_string(length0)
verify_data(dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
backup = create_backup(address, snap1, backup_target)["URL"]
cmd.restore_to_file(address, backup, "", output_raw_path, IMAGE_FORMAT_RAW)
output1_raw = read_file(output_raw_path, offset0, length0)
assert output1_raw == snap1_data
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, "", output_qcow2_path,
IMAGE_FORMAT_QCOW2)
output1_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
assert output1_qcow2.decode('utf-8') == snap1_data
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_NAME, snap0)
rm_snaps(address, [snap1])
rm_backups(address, ENGINE_NAME, [backup])
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2)
snap1_data = random_string(length0)
verify_data(dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
snap2_data = random_string(length1)
verify_data(dev, offset0, snap2_data)
snap2 = cmd.snapshot_create(address)
backup = create_backup(address, snap2, backup_target)["URL"]
cmd.restore_to_file(address, backup, "", output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(output_raw_path, offset0, length1)
output2_raw_snap1 = read_file(output_raw_path, offset1, length0 - length1)
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset1:length0]
cmd.restore_to_file(address, backup, "", output_qcow2_path,
IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - length1)
assert output2_qcow2_snap2.decode('utf-8') == snap2_data
assert output2_qcow2_snap1.decode('utf-8') == snap1_data[offset1:length0]
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_NAME, snap0)
rm_snaps(address, [snap1, snap2])
rm_backups(address, ENGINE_NAME, [backup])
