def rm_backups(*backups):
for b in backups:
cmd.backup_rm(b)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(b)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(b)
def test_backup_with_backing_file(backing_dev): # NOQA
dev = backing_dev # NOQA
offset = 0
length = 256
snap0 = cmd.snapshot_create()
before = read_dev(dev, offset, length)
assert before != ""
snap0_checksum = common.checksum_dev(dev)
exists = read_from_backing_file(offset, length)
assert before == exists
backup0 = cmd.backup_create(snap0, BACKUP_DEST)
backup0_info = cmd.backup_inspect(backup0)
assert backup0_info["URL"] == backup0
assert backup0_info["VolumeName"] == VOLUME_NAME
assert backup0_info["VolumeSize"] == VOLUME_SIZE
assert snap0 in backup0_info["SnapshotName"]
test_backup(dev)
cmd.backup_restore(backup0)
after = read_dev(dev, offset, length)
assert before == after
c = common.checksum_dev(dev)
assert c == snap0_checksum
cmd.backup_rm(backup0)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup0)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup0)
assert 'cannot find' in str(e.value)
def test_backup_with_backing_file(backing_dev): # NOQA
dev = backing_dev # NOQA
offset = 0
length = 256
snap0 = cmd.snapshot_create()
before = read_dev(dev, offset, length)
assert before != ""
exists = read_from_backing_file(offset, length)
assert before == exists
backup0 = cmd.backup_create(snap0, BACKUP_DEST)
backup0_info = cmd.backup_inspect(backup0)
assert backup0_info["BackupURL"] == backup0
assert backup0_info["VolumeName"] == VOLUME_NAME
assert backup0_info["VolumeSize"] == VOLUME_SIZE
assert snap0 in backup0_info["SnapshotName"]
test_backup(dev)
cmd.backup_restore(backup0)
after = read_dev(dev, offset, length)
assert before == after
cmd.backup_rm(backup0)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup0)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup0)
assert 'cannot find' in str(e.value)
def backup_with_backing_file_test(backing_dev, backup_target): # NOQA
dev = backing_dev # NOQA
offset = 0
length = 256
snap0 = cmd.snapshot_create()
before = read_dev(dev, offset, length)
assert before != ""
snap0_checksum = common.checksum_dev(dev)
exists = read_from_backing_file(offset, length)
assert before == exists
backup0 = cmd.backup_create(snap0, backup_target)
backup0_info = cmd.backup_inspect(backup0)
assert backup0_info["URL"] == backup0
assert backup0_info["VolumeName"] == VOLUME_NAME
assert backup0_info["VolumeSize"] == VOLUME_SIZE
assert snap0 in backup0_info["SnapshotName"]
backup_test(dev, backup_target)
cmd.backup_restore(backup0)
after = read_dev(dev, offset, length)
assert before == after
c = common.checksum_dev(dev)
assert c == snap0_checksum
cmd.backup_rm(backup0)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup0)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(backup0)
def rm_backups(backups, url=cmd.CONTROLLER):
for b in backups:
cmd.backup_rm(b, url)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(b)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(b)
def rm_backups(url, engine_name, backups):
for b in backups:
cmd.backup_rm(url, b)
with pytest.raises(subprocess.CalledProcessError):
restore_with_frontend(url, engine_name, b)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(url, b)
# Engine frontend is down, Start it up
start_engine_frontend(engine_name)
def create_backup(url, snap, backup_target):
backup = cmd.backup_create(url, snap, backup_target)
backup_info = cmd.backup_inspect(url, backup)
assert backup_info["URL"] == backup
assert backup_info["VolumeSize"] == SIZE_STR
assert snap in backup_info["SnapshotName"]
return backup_info
def create_backup(backup_target, snap):
backup = cmd.backup_create(snap, backup_target)
backup_info = cmd.backup_inspect(backup)
assert backup_info["URL"] == backup
assert backup_info["VolumeName"] == VOLUME_NAME
assert backup_info["VolumeSize"] == VOLUME_SIZE
assert backup_info["SnapshotName"]
return backup
def test_backup(dev): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1_checksum = common.checksum_dev(dev)
snap1 = cmd.snapshot_create()
backup1 = cmd.backup_create(snap1, BACKUP_DEST)
backup1_info = cmd.backup_inspect(backup1)
assert backup1_info["URL"] == backup1
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["VolumeSize"] == VOLUME_SIZE
assert backup1_info["Size"] == BLOCK_SIZE
assert snap1 in backup1_info["SnapshotName"]
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2_checksum = common.checksum_dev(dev)
snap2 = cmd.snapshot_create()
backup2 = cmd.backup_create(snap2, BACKUP_DEST)
backup2_info = cmd.backup_inspect(backup2)
assert backup2_info["URL"] == backup2
assert backup2_info["VolumeName"] == VOLUME_NAME
assert backup2_info["VolumeSize"] == VOLUME_SIZE
assert backup2_info["Size"] == BLOCK_SIZE
assert snap2 in backup2_info["SnapshotName"]
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3_checksum = common.checksum_dev(dev)
snap3 = cmd.snapshot_create()
backup3 = cmd.backup_create(snap3, BACKUP_DEST)
backup3_info = cmd.backup_inspect(backup3)
assert backup3_info["URL"] == backup3
assert backup3_info["VolumeName"] == VOLUME_NAME
assert backup3_info["VolumeSize"] == VOLUME_SIZE
assert backup3_info["Size"] == BLOCK_SIZE
assert snap3 in backup3_info["SnapshotName"]
cmd.backup_restore(backup3)
readed = read_dev(dev, offset, length)
assert readed == snap3_data
c = common.checksum_dev(dev)
assert c == snap3_checksum
cmd.backup_rm(backup3)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup3)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup3)
assert 'cannot find' in str(e.value)
cmd.backup_restore(backup1)
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = common.checksum_dev(dev)
assert c == snap1_checksum
cmd.backup_rm(backup1)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup1)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup1)
assert 'cannot find' in str(e.value)
cmd.backup_restore(backup2)
readed = read_dev(dev, offset, length)
assert readed == snap2_data
c = common.checksum_dev(dev)
assert c == snap2_checksum
cmd.backup_rm(backup2)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup2)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup2)
assert 'cannot find' in str(e.value)
def test_backup(dev): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1 = cmd.snapshot_create()
backup1 = cmd.backup_create(snap1, BACKUP_DEST)
backup1_info = cmd.backup_inspect(backup1)
assert backup1_info["BackupURL"] == backup1
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["VolumeSize"] == VOLUME_SIZE
assert snap1 in backup1_info["SnapshotName"]
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create()
backup2 = cmd.backup_create(snap2, BACKUP_DEST)
backup2_info = cmd.backup_inspect(backup2)
assert backup2_info["BackupURL"] == backup2
assert backup2_info["VolumeName"] == VOLUME_NAME
assert backup2_info["VolumeSize"] == VOLUME_SIZE
assert snap2 in backup2_info["SnapshotName"]
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3 = cmd.snapshot_create()
backup3 = cmd.backup_create(snap3, BACKUP_DEST)
backup3_info = cmd.backup_inspect(backup3)
assert backup3_info["BackupURL"] == backup3
assert backup3_info["VolumeName"] == VOLUME_NAME
assert backup3_info["VolumeSize"] == VOLUME_SIZE
assert snap3 in backup3_info["SnapshotName"]
cmd.backup_restore(backup3)
readed = read_dev(dev, offset, length)
assert readed == snap3_data
cmd.backup_rm(backup3)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup3)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup3)
assert 'cannot find' in str(e.value)
cmd.backup_restore(backup1)
readed = read_dev(dev, offset, length)
assert readed == snap1_data
cmd.backup_rm(backup1)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup1)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup1)
assert 'cannot find' in str(e.value)
cmd.backup_restore(backup2)
readed = read_dev(dev, offset, length)
assert readed == snap2_data
cmd.backup_rm(backup2)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup2)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup2)
assert 'cannot find' in str(e.value)
def backup_test(dev, backup_target): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1_checksum = common.checksum_dev(dev)
snap1 = cmd.snapshot_create()
backup1 = cmd.backup_create(snap1, backup_target)
backup1_info = cmd.backup_inspect(backup1)
assert backup1_info["URL"] == backup1
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["VolumeSize"] == VOLUME_SIZE
assert backup1_info["Size"] == BLOCK_SIZE
assert snap1 in backup1_info["SnapshotName"]
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2_checksum = common.checksum_dev(dev)
snap2 = cmd.snapshot_create()
backup2 = cmd.backup_create(snap2, backup_target)
backup2_info = cmd.backup_inspect(backup2)
assert backup2_info["URL"] == backup2
assert backup2_info["VolumeName"] == VOLUME_NAME
assert backup2_info["VolumeSize"] == VOLUME_SIZE
assert backup2_info["Size"] == BLOCK_SIZE
assert snap2 in backup2_info["SnapshotName"]
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3_checksum = common.checksum_dev(dev)
snap3 = cmd.snapshot_create()
backup3 = cmd.backup_create(snap3, backup_target)
backup3_info = cmd.backup_inspect(backup3)
assert backup3_info["URL"] == backup3
assert backup3_info["VolumeName"] == VOLUME_NAME
assert backup3_info["VolumeSize"] == VOLUME_SIZE
assert backup3_info["Size"] == BLOCK_SIZE
assert snap3 in backup3_info["SnapshotName"]
cmd.backup_restore(backup3)
readed = read_dev(dev, offset, length)
assert readed == snap3_data
c = common.checksum_dev(dev)
assert c == snap3_checksum
cmd.backup_rm(backup3)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup3)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(backup3)
cmd.backup_restore(backup1)
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = common.checksum_dev(dev)
assert c == snap1_checksum
cmd.backup_rm(backup1)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup1)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(backup1)
cmd.backup_restore(backup2)
readed = read_dev(dev, offset, length)
assert readed == snap2_data
c = common.checksum_dev(dev)
assert c == snap2_checksum
cmd.backup_rm(backup2)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup2)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(backup2)
def restore_inc_test(
grpc_engine_manager, # NOQA
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2, # NOQA
grpc_dr_controller, # NOQA
grpc_dr_replica1,
grpc_dr_replica2, # NOQA
backup_target): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create(address)
backup0 = create_backup(address, snap0, backup_target)["URL"]
backup0_name = cmd.backup_inspect(address, backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create(address)
backup1 = create_backup(address, snap1, backup_target)["URL"]
backup1_name = cmd.backup_inspect(address, backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create(address)
backup2 = create_backup(address, snap2, backup_target)["URL"]
backup2_name = cmd.backup_inspect(address, backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create(address)
backup3 = create_backup(address, snap3, backup_target)["URL"]
backup3_name = cmd.backup_inspect(address, backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create(address)
backup4 = create_backup(address, snap4, backup_target)["URL"]
backup4_name = cmd.backup_inspect(address, backup4)['Name']
# start no-frontend volume
# start dr volume (no frontend)
dr_address = grpc_dr_controller.address
start_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
cmd.backup_restore(dr_address, backup0)
wait_for_restore_completion(dr_address, backup0)
verify_no_frontend_data(grpc_engine_manager, 0, snap0_data,
grpc_dr_controller)
# mock restore crash/error
delta_file1 = "volume-delta-" + backup0_name + ".img"
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk + ".tmp"]
subprocess.check_output(command).strip()
# should fail
is_failed = False
cmd.restore_inc(dr_address, backup1, backup0_name)
for i in range(RETRY_COUNTS):
rs = cmd.restore_status(dr_address)
for status in rs.values():
if status['backupURL'] != backup1:
break
if 'error' in status.keys():
if status['error'] != "":
assert 'no such file or directory' in \
status['error']
is_failed = True
if is_failed:
break
time.sleep(RETRY_INTERVAL)
assert is_failed
assert path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert path.exists(FIXED_REPLICA_PATH2 + delta_file1)
for blk in blocks:
command = ["mv", blk + ".tmp", blk]
subprocess.check_output(command)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
# race condition: last restoration has failed
# but `isRestoring` hasn't been cleanup
for i in range(RETRY_COUNTS):
try:
restore_incrementally(dr_address, backup1, backup0_name)
break
except subprocess.CalledProcessError as e:
if "already in progress" not in e.output:
time.sleep(RETRY_INTERVAL)
else:
raise e
verify_no_frontend_data(grpc_engine_manager, 0, data1, grpc_dr_controller)
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup1_name)
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
restore_incrementally(dr_address, backup2, backup1_name)
verify_no_frontend_data(grpc_engine_manager, 0, data2, grpc_dr_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup2_name)
# mock race condition
with pytest.raises(subprocess.CalledProcessError) as e:
restore_incrementally(dr_address, backup1, backup0_name)
assert "doesn't match lastRestored" in e
data3 = zero_string * length3 + data2[length3:length2]
restore_incrementally(dr_address, backup3, backup2_name)
verify_no_frontend_data(grpc_engine_manager, 0, data3, grpc_dr_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3)
assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup3_name)
# mock corner case: invalid last-restored backup
rm_backups(address, ENGINE_NAME, [backup3])
# actually it is full restoration
restore_incrementally(dr_address, backup4, backup3_name)
verify_no_frontend_data(grpc_engine_manager, 0, snap4_data,
grpc_dr_controller)
status = cmd.restore_status(dr_address)
compare_last_restored_with_backup(status, backup4_name)
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*tempoary"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(grpc_engine_manager, grpc_dr_controller,
grpc_dr_replica1, grpc_dr_replica2)
rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4])
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def restore_inc_test(
controller,
replica1,
replica2, # NOQA
sb_controller,
sb_replica1,
sb_replica2,
backup_target): # NOQA
launcher.start_engine_frontend(FRONTEND_TGT_BLOCKDEV, url=LAUNCHER)
dev = common.get_dev(replica1, replica2, controller)
zero_string = b'\x00'.decode('utf-8')
# backup0: 256 random data in 1st block
length0 = 256
snap0_data = common.random_string(length0)
verify_data(dev, 0, snap0_data)
verify_data(dev, BLOCK_SIZE, snap0_data)
snap0 = cmd.snapshot_create()
backup0 = create_backup(backup_target, snap0)
backup0_name = cmd.backup_inspect(backup0)['Name']
# backup1: 32 random data + 32 zero data + 192 random data in 1st block
length1 = 32
offset1 = 32
snap1_data = zero_string * length1
verify_data(dev, offset1, snap1_data)
snap1 = cmd.snapshot_create()
backup1 = create_backup(backup_target, snap1)
backup1_name = cmd.backup_inspect(backup1)['Name']
# backup2: 32 random data + 256 random data in 1st block,
# 256 random data in 2nd block
length2 = 256
offset2 = 32
snap2_data = common.random_string(length2)
verify_data(dev, offset2, snap2_data)
verify_data(dev, BLOCK_SIZE, snap2_data)
snap2 = cmd.snapshot_create()
backup2 = create_backup(backup_target, snap2)
backup2_name = cmd.backup_inspect(backup2)['Name']
# backup3: 64 zero data + 192 random data in 1st block
length3 = 64
offset3 = 0
verify_data(dev, offset3, zero_string * length3)
verify_data(dev, length2, zero_string * offset2)
verify_data(dev, BLOCK_SIZE, zero_string * length2)
snap3 = cmd.snapshot_create()
backup3 = create_backup(backup_target, snap3)
backup3_name = cmd.backup_inspect(backup3)['Name']
# backup4: 256 random data in 1st block
length4 = 256
offset4 = 0
snap4_data = common.random_string(length4)
verify_data(dev, offset4, snap4_data)
snap4 = cmd.snapshot_create()
backup4 = create_backup(backup_target, snap4)
backup4_name = cmd.backup_inspect(backup4)['Name']
common.cleanup_replica(replica1)
common.cleanup_replica(replica2)
common.cleanup_controller(controller)
launcher.shutdown_engine_frontend(url=LAUNCHER)
# start no-frontend volume
# start standby volume (no frontend)
start_no_frontend_volume(sb_controller, sb_replica1, sb_replica2)
restore_for_no_frontend_volume(backup0, sb_controller)
verify_no_frontend_data(0, snap0_data, sb_controller)
# mock restore crash/error
delta_file1 = "volume-delta-" + backup0_name + ".img"
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*blk"]
blocks = subprocess.check_output(command).split()
assert len(blocks) != 0
for blk in blocks:
command = ["mv", blk, blk + ".tmp"]
subprocess.check_output(command).strip()
with pytest.raises(subprocess.CalledProcessError):
cmd.restore_inc(backup1, backup0_name, CONTROLLER_NO_FRONTEND)
assert path.exists(STANDBY_REPLICA1_PATH + delta_file1)
assert path.exists(STANDBY_REPLICA2_PATH + delta_file1)
for blk in blocks:
command = ["mv", blk + ".tmp", blk]
subprocess.check_output(command)
data1 = \
snap0_data[0:offset1] + snap1_data + \
snap0_data[offset1+length1:]
cmd.restore_inc(backup1, backup0_name, CONTROLLER_NO_FRONTEND)
verify_no_frontend_data(0, data1, sb_controller)
assert not path.exists(STANDBY_REPLICA1_PATH + delta_file1)
assert not path.exists(STANDBY_REPLICA2_PATH + delta_file1)
volume_info = cmd.info(CONTROLLER_NO_FRONTEND)
assert volume_info['lastRestored'] == backup1_name
data2 = \
data1[0:offset2] + snap2_data + \
zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data
cmd.restore_inc(backup2, backup1_name, CONTROLLER_NO_FRONTEND)
verify_no_frontend_data(0, data2, sb_controller)
delta_file2 = "volume-delta-" + backup1_name + ".img"
assert not path.exists(STANDBY_REPLICA1_PATH + delta_file2)
assert not path.exists(STANDBY_REPLICA2_PATH + delta_file2)
volume_info = cmd.info(CONTROLLER_NO_FRONTEND)
assert volume_info['lastRestored'] == backup2_name
# mock race condition
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.restore_inc(backup1, backup0_name, CONTROLLER_NO_FRONTEND)
assert "doesn't match lastRestored" in e
data3 = zero_string * length3 + data2[length3:length2]
cmd.restore_inc(backup3, backup2_name, CONTROLLER_NO_FRONTEND)
verify_no_frontend_data(0, data3, sb_controller)
delta_file3 = "volume-delta-" + backup3_name + ".img"
assert not path.exists(STANDBY_REPLICA1_PATH + delta_file3)
assert not path.exists(STANDBY_REPLICA2_PATH + delta_file3)
volume_info = cmd.info(CONTROLLER_NO_FRONTEND)
assert volume_info['lastRestored'] == backup3_name
# mock corner case: invalid last-restored backup
rm_backups([backup3])
# actually it is full restoration
cmd.restore_inc(backup4, backup3_name, CONTROLLER_NO_FRONTEND)
verify_no_frontend_data(0, snap4_data, sb_controller)
volume_info = cmd.info(CONTROLLER_NO_FRONTEND)
assert volume_info['lastRestored'] == backup4_name
if "vfs" in backup_target:
command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME]
backup_volume_path = subprocess.check_output(command).strip()
command = ["find", backup_volume_path, "-name", "*tempoary"]
tmp_files = subprocess.check_output(command).split()
assert len(tmp_files) == 0
cleanup_no_frontend_volume(sb_controller, sb_replica1, sb_replica2)
rm_backups([backup0, backup1, backup2, backup4])