def test_inc_restore_failure_invalid_block(grpc_controller, grpc_replica1,
grpc_replica2,
grpc_controller_no_frontend,
grpc_fixed_dir_replica1,
grpc_fixed_dir_replica2,
backup_targets): # NOQA
# This case is for vfs backup only
for backup_target in backup_targets:
if "vfs" in backup_target:
break
assert backup_target
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
# start dr volume (no frontend)
dr_address = grpc_controller_no_frontend.address
start_no_frontend_volume(grpc_controller_no_frontend,
grpc_fixed_dir_replica1, grpc_fixed_dir_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(0, snap0_data, grpc_controller_no_frontend)
# mock inc restore error: invalid block
delta_file = "volume-delta-" + backup0_name + ".img"
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk + ".tmp".encode('utf-8')]
subprocess.check_output(command).strip()
cmd.backup_restore(dr_address, backup1)
# restore status should contain the error info
failed_restore, finished_restore = 0, 0
for i in range(RETRY_COUNTS):
failed_restore, finished_restore = 0, 0
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
failed_restore += 1
if not status["isRestoring"]:
finished_restore += 1
if failed_restore == 2 and finished_restore == 2:
break
time.sleep(RETRY_INTERVAL)
assert failed_restore == 2 and finished_restore == 2
assert path.exists(FIXED_REPLICA_PATH1 + delta_file)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file)
for blk in blocks:
command = ["mv", blk + ".tmp".encode('utf-8'), blk]
subprocess.check_output(command)
cleanup_no_frontend_volume(grpc_controller_no_frontend,
grpc_fixed_dir_replica1,
grpc_fixed_dir_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1])
cmd.sync_agent_server_reset(address)
cmd.sync_agent_server_reset(dr_address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def inc_restore_failure_cleanup_error_test(grpc_controller, grpc_replica1,
grpc_replica2, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2,
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(0, snap0_data, grpc_dr_controller)
# mock inc restore crash/error: cannot clean up the delta file path
delta_file = "volume-delta-" + backup0_name + ".img"
command = ["mkdir", "-p", FIXED_REPLICA_PATH1 + delta_file + "/dir"]
subprocess.check_output(command).strip()
command = ["mkdir", "-p", FIXED_REPLICA_PATH2 + delta_file + "/dir"]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup1)
assert "failed to clean up the existing file" in e.value.stdout
command = ["rm", "-r", FIXED_REPLICA_PATH1 + delta_file]
subprocess.check_output(command).strip()
command = ["rm", "-r", FIXED_REPLICA_PATH2 + delta_file]
subprocess.check_output(command).strip()
# the restore status will be reverted/keep unchanged
# if an error is triggered before the actual restore is performed
rs = cmd.restore_status(dr_address)
for status in rs.values():
assert not status["isRestoring"]
assert status['backupURL'] == backup0
assert 'error' not in status
assert status['progress'] == 100
assert status['state'] == "complete"
cleanup_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1])
cmd.sync_agent_server_reset(address)
cmd.sync_agent_server_reset(dr_address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def restore_inc_test(grpc_controller, grpc_replica1, grpc_replica2,
grpc_dr_controller, grpc_dr_replica1, grpc_dr_replica2,
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
# mock restore crash/error:
# By adding attribute `immutable`, Longhorn cannot create a file
# for the restore. Then the following restore command will fail.
command = ["chattr", "+i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "+i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup0)
assert "operation not permitted" in e.value.stdout
# the restore status will be reverted/keep unchanged
# if an error is triggered before the actual restore is performed
rs = cmd.restore_status(dr_address)
for status in rs.values():
assert not status["isRestoring"]
assert not status['backupURL']
assert 'error' not in status
assert 'progress' not in status
assert not status['state']
command = ["chattr", "-i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "-i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
cmd.sync_agent_server_reset(dr_address)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(0, snap0_data, grpc_dr_controller)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
cmd.backup_restore(dr_address, backup1)
wait_for_restore_completion(dr_address, backup1)
verify_no_frontend_data(0, data1, grpc_dr_controller)
delta_file1 = "volume-delta-" + backup0_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
cmd.backup_restore(dr_address, backup2)
wait_for_restore_completion(dr_address, backup2)
verify_no_frontend_data(0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition: duplicate inc restore calls
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup2)
wait_for_restore_completion(dr_address, backup2)
assert "already restored backup" in e.value.stdout
data3 = zero_string * length3 + data2[length3:length2]
cmd.backup_restore(dr_address, backup3)
wait_for_restore_completion(dr_address, backup3)
verify_no_frontend_data(0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# This inc restore will fall back to full restore
cmd.backup_restore(dr_address, backup4)
wait_for_restore_completion(dr_address, backup4)
verify_no_frontend_data(0, snap4_data, grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
# check if the tmp files during this special full restore are cleaned up.
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*snap_tmp"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_dr_controller, grpc_dr_replica1,
grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cmd.sync_agent_server_reset(dr_address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def restore_inc_test(
grpc_engine_manager, # NOQA
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2, # NOQA
grpc_dr_controller, # NOQA
grpc_dr_replica1,
grpc_dr_replica2, # NOQA
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
# mock restore crash/error:
# By adding attribute `immutable`, Longhorn cannot create a file
# for the restore. Then the following restore command will fail.
command = ["chattr", "+i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "+i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup0)
assert "operation not permitted" in e.value.stdout
command = ["chattr", "-i", FIXED_REPLICA_PATH1]
subprocess.check_output(command).strip()
command = ["chattr", "-i", FIXED_REPLICA_PATH2]
subprocess.check_output(command).strip()
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(grpc_engine_manager, 0, snap0_data,
grpc_dr_controller)
delta_file1 = "volume-delta-" + backup0_name + ".img"
# mock inc restore crash/error: cannot clean up the delta file path
command = ["mkdir", "-p", FIXED_REPLICA_PATH1 + delta_file1 + "/dir"]
subprocess.check_output(command).strip()
command = ["mkdir", "-p", FIXED_REPLICA_PATH2 + delta_file1 + "/dir"]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError):
cmd.restore_inc(dr_address, backup1, backup0_name)
command = ["rm", "-r", FIXED_REPLICA_PATH1 + delta_file1]
subprocess.check_output(command).strip()
command = ["rm", "-r", FIXED_REPLICA_PATH2 + delta_file1]
subprocess.check_output(command).strip()
if "vfs" in backup_target:
# restore status should contain the error info
failed_restore, finished_restore = 0, 0
for i in range(RETRY_COUNTS):
failed_restore, finished_restore = 0, 0
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'failed to clean up the existing file' in\
status['error']
failed_restore += 1
if not status["isRestoring"]:
finished_restore += 1
if failed_restore == 2 and finished_restore == 2:
break
time.sleep(RETRY_INTERVAL)
assert failed_restore == 2 and finished_restore == 2
# mock inc restore error: invalid block
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk + ".tmp".encode('utf-8')]
subprocess.check_output(command).strip()
cmd.restore_inc(dr_address, backup1, backup0_name)
# restore status should contain the error info
failed_restore, finished_restore = 0, 0
for i in range(RETRY_COUNTS):
failed_restore, finished_restore = 0, 0
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
failed_restore += 1
if not status["isRestoring"]:
finished_restore += 1
if failed_restore == 2 and finished_restore == 2:
break
time.sleep(RETRY_INTERVAL)
assert failed_restore == 2 and finished_restore == 2
assert path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file1)
for blk in blocks:
command = ["mv", blk + ".tmp".encode('utf-8'), blk]
subprocess.check_output(command)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
restore_incrementally(dr_address, backup1, backup0_name)
verify_no_frontend_data(grpc_engine_manager, 0, data1, grpc_dr_controller)
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
restore_incrementally(dr_address, backup2, backup1_name)
verify_no_frontend_data(grpc_engine_manager, 0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition
with pytest.raises(subprocess.CalledProcessError) as e:
restore_incrementally(dr_address, backup1, backup0_name)
assert "doesn't match field LastRestored" in e.value.stdout
data3 = zero_string * length3 + data2[length3:length2]
restore_incrementally(dr_address, backup3, backup2_name)
verify_no_frontend_data(grpc_engine_manager, 0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# actually it is full restoration
restore_incrementally(dr_address, backup4, backup3_name)
verify_no_frontend_data(grpc_engine_manager, 0, snap4_data,
grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*tempoary"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def backup_core(
bin,
engine_manager_client, # NOQA
grpc_controller_client, # NOQA
grpc_replica_client, # NOQA
grpc_replica_client2, # NOQA
backup_target):
open_replica(grpc_replica_client)
open_replica(grpc_replica_client2)
r1_url = grpc_replica_client.url
r2_url = grpc_replica_client2.url
v = grpc_controller_client.volume_start(replicas=[
r1_url,
r2_url,
])
assert v.replicaCount == 2
backup_type = urlparse(backup_target).scheme
# create & process backup1
snapshot1 = cmd.snapshot_create(grpc_controller_client.address)
output = grpc_replica_client.replica_get().chain[1]
assert output == 'volume-snap-{}.img'.format(snapshot1)
backup1 = cmd.backup_create(grpc_controller_client.address, snapshot1,
backup_target, {
'name': 'backup1',
'type': backup_type
})
backup1_info = cmd.backup_inspect(grpc_controller_client.address, backup1)
assert backup1_info["URL"] == backup1
assert backup1_info["IsIncremental"] is False
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["VolumeSize"] == SIZE_STR
assert backup1_info["SnapshotName"] == snapshot1
assert len(backup1_info["Labels"]) == 2
assert backup1_info["Labels"]["name"] == "backup1"
assert backup1_info["Labels"]["type"] == backup_type
# create & process backup2
snapshot2 = cmd.snapshot_create(grpc_controller_client.address)
output = grpc_replica_client.replica_get().chain[1]
assert output == 'volume-snap-{}.img'.format(snapshot2)
backup2 = cmd.backup_create(grpc_controller_client.address, snapshot2,
backup_target)
backup2_info = cmd.backup_inspect(grpc_controller_client.address, backup2)
assert backup2_info["URL"] == backup2
assert backup2_info["IsIncremental"] is True
assert backup2_info["VolumeName"] == VOLUME_NAME
assert backup2_info["VolumeSize"] == SIZE_STR
assert backup2_info["SnapshotName"] == snapshot2
if backup2_info["Labels"] is not None:
assert len(backup2_info["Labels"]) == 0
# list all known backups for volume
volume_info = cmd.backup_volume_list(
grpc_controller_client.address,
VOLUME_NAME,
backup_target,
include_backup_details=True)[VOLUME_NAME]
assert volume_info["Name"] == VOLUME_NAME
assert volume_info["Size"] == SIZE_STR
backup_list = volume_info["Backups"]
assert backup_list[backup1]["URL"] == backup1_info["URL"]
assert backup_list[backup1]["SnapshotName"] == backup1_info["SnapshotName"]
assert backup_list[backup1]["Size"] == backup1_info["Size"]
assert backup_list[backup1]["Created"] == backup1_info["Created"]
assert backup_list[backup1]["Messages"] is None
assert backup_list[backup2]["URL"] == backup2_info["URL"]
assert backup_list[backup2]["SnapshotName"] == backup2_info["SnapshotName"]
assert backup_list[backup2]["Size"] == backup2_info["Size"]
assert backup_list[backup2]["Created"] == backup2_info["Created"]
assert backup_list[backup2]["Messages"] is None
# test that corrupt backups are signaled during a list operation
# https://github.com/longhorn/longhorn/issues/1212
volume_dir = finddir(BACKUP_DIR, VOLUME_NAME)
assert volume_dir
assert os.path.exists(volume_dir)
backup_dir = os.path.join(volume_dir, "backups")
assert os.path.exists(backup_dir)
backup_cfg_name = "backup_" + backup2_info["Name"] + ".cfg"
assert backup_cfg_name
backup_cfg_path = findfile(backup_dir, backup_cfg_name)
assert os.path.exists(backup_cfg_path)
backup_tmp_cfg_path = os.path.join(volume_dir, backup_cfg_name)
os.rename(backup_cfg_path, backup_tmp_cfg_path)
assert os.path.exists(backup_tmp_cfg_path)
corrupt_backup = open(backup_cfg_path, "w")
assert corrupt_backup
assert corrupt_backup.write("{corrupt: definitely") > 0
corrupt_backup.close()
# request the new backup list
volume_info = cmd.backup_volume_list(
grpc_controller_client.address,
VOLUME_NAME,
backup_target,
include_backup_details=True)[VOLUME_NAME]
assert volume_info["Name"] == VOLUME_NAME
backup_list = volume_info["Backups"]
assert backup_list[backup1]["URL"] == backup1_info["URL"]
assert backup_list[backup1]["Messages"] is None
assert backup_list[backup2]["URL"] == backup2_info["URL"]
assert MESSAGE_TYPE_ERROR in backup_list[backup2]["Messages"]
# we still want to fail inspects, since they operate on urls
# with no guarantee of backup existence
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(grpc_controller_client.address, backup2)
# switch back to valid cfg
os.rename(backup_tmp_cfg_path, backup_cfg_path)
assert cmd.backup_inspect(grpc_controller_client.address, backup2)
# test that list returns a volume_info with an error message
# for a missing volume.cfg instead of failing with an error
# https://github.com/rancher/longhorn/issues/399
volume_cfg_path = findfile(volume_dir, VOLUME_CONFIG_FILE)
assert os.path.exists(volume_cfg_path)
volume_tmp_cfg_path = volume_cfg_path.replace(VOLUME_CONFIG_FILE,
VOLUME_TMP_CONFIG_FILE)
os.rename(volume_cfg_path, volume_tmp_cfg_path)
assert os.path.exists(volume_tmp_cfg_path)
volume_info = cmd.backup_volume_list(grpc_controller_client.address, "",
backup_target)
assert MESSAGE_TYPE_ERROR in volume_info[VOLUME_NAME]["Messages"]
os.rename(volume_tmp_cfg_path, volume_cfg_path)
assert os.path.exists(volume_cfg_path)
volume_info = cmd.backup_volume_list(grpc_controller_client.address, "",
backup_target)
assert volume_info[VOLUME_NAME]["Messages"] is not None
assert MESSAGE_TYPE_ERROR not in volume_info[VOLUME_NAME]["Messages"]
# backup doesn't exists so it should error
with pytest.raises(subprocess.CalledProcessError):
url = backup_target + "?backup=backup-unk" + "&volume=" + VOLUME_NAME
cmd.backup_inspect(grpc_controller_client.address, url)
# this returns unsupported driver since `bad` is not a known scheme
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(grpc_controller_client.address, "bad://xxx")
reset_volume(grpc_controller_client, grpc_replica_client,
grpc_replica_client2)
restore_with_frontend(grpc_controller_client.address, ENGINE_NAME, backup1)
restore_with_frontend(grpc_controller_client.address, ENGINE_NAME, backup2)
# remove backups + volume
cmd.backup_rm(grpc_controller_client.address, backup1)
cmd.backup_rm(grpc_controller_client.address, backup2)
cmd.backup_volume_rm(grpc_controller_client.address, VOLUME_NAME,
backup_target)
assert os.path.exists(BACKUP_DIR)
assert not os.path.exists(volume_cfg_path)
def restore_inc_test(grpc_engine_manager, # NOQA
grpc_controller, # NOQA
grpc_replica1, grpc_replica2, # NOQA
grpc_dr_controller, # NOQA
grpc_dr_replica1, grpc_dr_replica2, # NOQA
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_engine_manager,
grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(grpc_engine_manager,
0, snap0_data, grpc_dr_controller)
# mock restore crash/error
delta_file1 = "volume-delta-" + backup0_name + ".img"
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk+".tmp".encode('utf-8')]
subprocess.check_output(command).strip()
# should fail
is_failed = False
cmd.restore_inc(dr_address, backup1, backup0_name)
for i in range(RETRY_COUNTS):
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
is_failed = True
if is_failed:
break
time.sleep(RETRY_INTERVAL)
assert is_failed
assert path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file1)
for blk in blocks:
command = ["mv", blk+".tmp".encode('utf-8'), blk]
subprocess.check_output(command)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
# race condition: last restoration has failed
# but `isRestoring` hasn't been cleanup
for i in range(RETRY_COUNTS):
try:
restore_incrementally(dr_address, backup1, backup0_name)
break
except subprocess.CalledProcessError as e:
if "already in progress" not in e.output:
time.sleep(RETRY_INTERVAL)
else:
raise e
verify_no_frontend_data(grpc_engine_manager,
0, data1, grpc_dr_controller)
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
restore_incrementally(dr_address, backup2, backup1_name)
verify_no_frontend_data(grpc_engine_manager,
0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition
with pytest.raises(subprocess.CalledProcessError) as e:
restore_incrementally(dr_address, backup1, backup0_name)
assert "doesn't match lastRestored" in e
data3 = zero_string * length3 + data2[length3:length2]
restore_incrementally(dr_address, backup3, backup2_name)
verify_no_frontend_data(grpc_engine_manager,
0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# actually it is full restoration
restore_incrementally(dr_address, backup4, backup3_name)
verify_no_frontend_data(grpc_engine_manager,
0, snap4_data, grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*tempoary"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_engine_manager,
grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
