def rm_backups(*backups): for b in backups: cmd.backup_rm(b) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(b) with pytest.raises(subprocess.CalledProcessError): cmd.backup_inspect(b)
def test_backup_with_backing_file(backing_dev): # NOQA dev = backing_dev # NOQA offset = 0 length = 256 snap0 = cmd.snapshot_create() before = read_dev(dev, offset, length) assert before != "" snap0_checksum = common.checksum_dev(dev) exists = read_from_backing_file(offset, length) assert before == exists backup0 = cmd.backup_create(snap0, BACKUP_DEST) backup0_info = cmd.backup_inspect(backup0) assert backup0_info["URL"] == backup0 assert backup0_info["VolumeName"] == VOLUME_NAME assert backup0_info["VolumeSize"] == VOLUME_SIZE assert snap0 in backup0_info["SnapshotName"] test_backup(dev) cmd.backup_restore(backup0) after = read_dev(dev, offset, length) assert before == after c = common.checksum_dev(dev) assert c == snap0_checksum cmd.backup_rm(backup0) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup0) with pytest.raises(subprocess.CalledProcessError) as e: cmd.backup_inspect(backup0) assert 'cannot find' in str(e.value)
def test_backup_with_backing_file(backing_dev): # NOQA dev = backing_dev # NOQA offset = 0 length = 256 snap0 = cmd.snapshot_create() before = read_dev(dev, offset, length) assert before != "" exists = read_from_backing_file(offset, length) assert before == exists backup0 = cmd.backup_create(snap0, BACKUP_DEST) backup0_info = cmd.backup_inspect(backup0) assert backup0_info["BackupURL"] == backup0 assert backup0_info["VolumeName"] == VOLUME_NAME assert backup0_info["VolumeSize"] == VOLUME_SIZE assert snap0 in backup0_info["SnapshotName"] test_backup(dev) cmd.backup_restore(backup0) after = read_dev(dev, offset, length) assert before == after cmd.backup_rm(backup0) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup0) with pytest.raises(subprocess.CalledProcessError) as e: cmd.backup_inspect(backup0) assert 'cannot find' in str(e.value)
def backup_with_backing_file_test(backing_dev, backup_target): # NOQA dev = backing_dev # NOQA offset = 0 length = 256 snap0 = cmd.snapshot_create() before = read_dev(dev, offset, length) assert before != "" snap0_checksum = common.checksum_dev(dev) exists = read_from_backing_file(offset, length) assert before == exists backup0 = cmd.backup_create(snap0, backup_target) backup0_info = cmd.backup_inspect(backup0) assert backup0_info["URL"] == backup0 assert backup0_info["VolumeName"] == VOLUME_NAME assert backup0_info["VolumeSize"] == VOLUME_SIZE assert snap0 in backup0_info["SnapshotName"] backup_test(dev, backup_target) cmd.backup_restore(backup0) after = read_dev(dev, offset, length) assert before == after c = common.checksum_dev(dev) assert c == snap0_checksum cmd.backup_rm(backup0) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup0) with pytest.raises(subprocess.CalledProcessError): cmd.backup_inspect(backup0)
def rm_backups(backups, url=cmd.CONTROLLER): for b in backups: cmd.backup_rm(b, url) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(b) with pytest.raises(subprocess.CalledProcessError): cmd.backup_inspect(b)
def rm_backups(url, engine_name, backups): for b in backups: cmd.backup_rm(url, b) with pytest.raises(subprocess.CalledProcessError): restore_with_frontend(url, engine_name, b) with pytest.raises(subprocess.CalledProcessError): cmd.backup_inspect(url, b) # Engine frontend is down, Start it up start_engine_frontend(engine_name)
def create_backup(url, snap, backup_target): backup = cmd.backup_create(url, snap, backup_target) backup_info = cmd.backup_inspect(url, backup) assert backup_info["URL"] == backup assert backup_info["VolumeSize"] == SIZE_STR assert snap in backup_info["SnapshotName"] return backup_info
def create_backup(backup_target, snap): backup = cmd.backup_create(snap, backup_target) backup_info = cmd.backup_inspect(backup) assert backup_info["URL"] == backup assert backup_info["VolumeName"] == VOLUME_NAME assert backup_info["VolumeSize"] == VOLUME_SIZE assert backup_info["SnapshotName"] return backup
def test_backup(dev): # NOQA offset = 0 length = 128 snap1_data = common.random_string(length) common.verify_data(dev, offset, snap1_data) snap1_checksum = common.checksum_dev(dev) snap1 = cmd.snapshot_create() backup1 = cmd.backup_create(snap1, BACKUP_DEST) backup1_info = cmd.backup_inspect(backup1) assert backup1_info["URL"] == backup1 assert backup1_info["VolumeName"] == VOLUME_NAME assert backup1_info["VolumeSize"] == VOLUME_SIZE assert backup1_info["Size"] == BLOCK_SIZE assert snap1 in backup1_info["SnapshotName"] snap2_data = common.random_string(length) common.verify_data(dev, offset, snap2_data) snap2_checksum = common.checksum_dev(dev) snap2 = cmd.snapshot_create() backup2 = cmd.backup_create(snap2, BACKUP_DEST) backup2_info = cmd.backup_inspect(backup2) assert backup2_info["URL"] == backup2 assert backup2_info["VolumeName"] == VOLUME_NAME assert backup2_info["VolumeSize"] == VOLUME_SIZE assert backup2_info["Size"] == BLOCK_SIZE assert snap2 in backup2_info["SnapshotName"] snap3_data = common.random_string(length) common.verify_data(dev, offset, snap3_data) snap3_checksum = common.checksum_dev(dev) snap3 = cmd.snapshot_create() backup3 = cmd.backup_create(snap3, BACKUP_DEST) backup3_info = cmd.backup_inspect(backup3) assert backup3_info["URL"] == backup3 assert backup3_info["VolumeName"] == VOLUME_NAME assert backup3_info["VolumeSize"] == VOLUME_SIZE assert backup3_info["Size"] == BLOCK_SIZE assert snap3 in backup3_info["SnapshotName"] cmd.backup_restore(backup3) readed = read_dev(dev, offset, length) assert readed == snap3_data c = common.checksum_dev(dev) assert c == snap3_checksum cmd.backup_rm(backup3) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup3) with pytest.raises(subprocess.CalledProcessError) as e: cmd.backup_inspect(backup3) assert 'cannot find' in str(e.value) cmd.backup_restore(backup1) readed = read_dev(dev, offset, length) assert readed == snap1_data c = common.checksum_dev(dev) assert c == snap1_checksum cmd.backup_rm(backup1) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup1) with pytest.raises(subprocess.CalledProcessError) as e: cmd.backup_inspect(backup1) assert 'cannot find' in str(e.value) cmd.backup_restore(backup2) readed = read_dev(dev, offset, length) assert readed == snap2_data c = common.checksum_dev(dev) assert c == snap2_checksum cmd.backup_rm(backup2) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup2) with pytest.raises(subprocess.CalledProcessError) as e: cmd.backup_inspect(backup2) assert 'cannot find' in str(e.value)
def test_backup(dev): # NOQA offset = 0 length = 128 snap1_data = common.random_string(length) common.verify_data(dev, offset, snap1_data) snap1 = cmd.snapshot_create() backup1 = cmd.backup_create(snap1, BACKUP_DEST) backup1_info = cmd.backup_inspect(backup1) assert backup1_info["BackupURL"] == backup1 assert backup1_info["VolumeName"] == VOLUME_NAME assert backup1_info["VolumeSize"] == VOLUME_SIZE assert snap1 in backup1_info["SnapshotName"] snap2_data = common.random_string(length) common.verify_data(dev, offset, snap2_data) snap2 = cmd.snapshot_create() backup2 = cmd.backup_create(snap2, BACKUP_DEST) backup2_info = cmd.backup_inspect(backup2) assert backup2_info["BackupURL"] == backup2 assert backup2_info["VolumeName"] == VOLUME_NAME assert backup2_info["VolumeSize"] == VOLUME_SIZE assert snap2 in backup2_info["SnapshotName"] snap3_data = common.random_string(length) common.verify_data(dev, offset, snap3_data) snap3 = cmd.snapshot_create() backup3 = cmd.backup_create(snap3, BACKUP_DEST) backup3_info = cmd.backup_inspect(backup3) assert backup3_info["BackupURL"] == backup3 assert backup3_info["VolumeName"] == VOLUME_NAME assert backup3_info["VolumeSize"] == VOLUME_SIZE assert snap3 in backup3_info["SnapshotName"] cmd.backup_restore(backup3) readed = read_dev(dev, offset, length) assert readed == snap3_data cmd.backup_rm(backup3) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup3) with pytest.raises(subprocess.CalledProcessError) as e: cmd.backup_inspect(backup3) assert 'cannot find' in str(e.value) cmd.backup_restore(backup1) readed = read_dev(dev, offset, length) assert readed == snap1_data cmd.backup_rm(backup1) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup1) with pytest.raises(subprocess.CalledProcessError) as e: cmd.backup_inspect(backup1) assert 'cannot find' in str(e.value) cmd.backup_restore(backup2) readed = read_dev(dev, offset, length) assert readed == snap2_data cmd.backup_rm(backup2) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup2) with pytest.raises(subprocess.CalledProcessError) as e: cmd.backup_inspect(backup2) assert 'cannot find' in str(e.value)
def backup_test(dev, backup_target): # NOQA offset = 0 length = 128 snap1_data = common.random_string(length) common.verify_data(dev, offset, snap1_data) snap1_checksum = common.checksum_dev(dev) snap1 = cmd.snapshot_create() backup1 = cmd.backup_create(snap1, backup_target) backup1_info = cmd.backup_inspect(backup1) assert backup1_info["URL"] == backup1 assert backup1_info["VolumeName"] == VOLUME_NAME assert backup1_info["VolumeSize"] == VOLUME_SIZE assert backup1_info["Size"] == BLOCK_SIZE assert snap1 in backup1_info["SnapshotName"] snap2_data = common.random_string(length) common.verify_data(dev, offset, snap2_data) snap2_checksum = common.checksum_dev(dev) snap2 = cmd.snapshot_create() backup2 = cmd.backup_create(snap2, backup_target) backup2_info = cmd.backup_inspect(backup2) assert backup2_info["URL"] == backup2 assert backup2_info["VolumeName"] == VOLUME_NAME assert backup2_info["VolumeSize"] == VOLUME_SIZE assert backup2_info["Size"] == BLOCK_SIZE assert snap2 in backup2_info["SnapshotName"] snap3_data = common.random_string(length) common.verify_data(dev, offset, snap3_data) snap3_checksum = common.checksum_dev(dev) snap3 = cmd.snapshot_create() backup3 = cmd.backup_create(snap3, backup_target) backup3_info = cmd.backup_inspect(backup3) assert backup3_info["URL"] == backup3 assert backup3_info["VolumeName"] == VOLUME_NAME assert backup3_info["VolumeSize"] == VOLUME_SIZE assert backup3_info["Size"] == BLOCK_SIZE assert snap3 in backup3_info["SnapshotName"] cmd.backup_restore(backup3) readed = read_dev(dev, offset, length) assert readed == snap3_data c = common.checksum_dev(dev) assert c == snap3_checksum cmd.backup_rm(backup3) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup3) with pytest.raises(subprocess.CalledProcessError): cmd.backup_inspect(backup3) cmd.backup_restore(backup1) readed = read_dev(dev, offset, length) assert readed == snap1_data c = common.checksum_dev(dev) assert c == snap1_checksum cmd.backup_rm(backup1) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup1) with pytest.raises(subprocess.CalledProcessError): cmd.backup_inspect(backup1) cmd.backup_restore(backup2) readed = read_dev(dev, offset, length) assert readed == snap2_data c = common.checksum_dev(dev) assert c == snap2_checksum cmd.backup_rm(backup2) with pytest.raises(subprocess.CalledProcessError): cmd.backup_restore(backup2) with pytest.raises(subprocess.CalledProcessError): cmd.backup_inspect(backup2)
def restore_inc_test( grpc_engine_manager, # NOQA grpc_controller, # NOQA grpc_replica1, grpc_replica2, # NOQA grpc_dr_controller, # NOQA grpc_dr_replica1, grpc_dr_replica2, # NOQA backup_target): # NOQA address = grpc_controller.address dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller) zero_string = b'\x00'.decode('utf-8') # backup0: 256 random data in 1st block length0 = 256 snap0_data = random_string(length0) verify_data(dev, 0, snap0_data) verify_data(dev, BLOCK_SIZE, snap0_data) snap0 = cmd.snapshot_create(address) backup0 = create_backup(address, snap0, backup_target)["URL"] backup0_name = cmd.backup_inspect(address, backup0)['Name'] # backup1: 32 random data + 32 zero data + 192 random data in 1st block length1 = 32 offset1 = 32 snap1_data = zero_string * length1 verify_data(dev, offset1, snap1_data) snap1 = cmd.snapshot_create(address) backup1 = create_backup(address, snap1, backup_target)["URL"] backup1_name = cmd.backup_inspect(address, backup1)['Name'] # backup2: 32 random data + 256 random data in 1st block, # 256 random data in 2nd block length2 = 256 offset2 = 32 snap2_data = random_string(length2) verify_data(dev, offset2, snap2_data) verify_data(dev, BLOCK_SIZE, snap2_data) snap2 = cmd.snapshot_create(address) backup2 = create_backup(address, snap2, backup_target)["URL"] backup2_name = cmd.backup_inspect(address, backup2)['Name'] # backup3: 64 zero data + 192 random data in 1st block length3 = 64 offset3 = 0 verify_data(dev, offset3, zero_string * length3) verify_data(dev, length2, zero_string * offset2) verify_data(dev, BLOCK_SIZE, zero_string * length2) snap3 = cmd.snapshot_create(address) backup3 = create_backup(address, snap3, backup_target)["URL"] backup3_name = cmd.backup_inspect(address, backup3)['Name'] # backup4: 256 random data in 1st block length4 = 256 offset4 = 0 snap4_data = random_string(length4) verify_data(dev, offset4, snap4_data) snap4 = cmd.snapshot_create(address) backup4 = create_backup(address, snap4, backup_target)["URL"] backup4_name = cmd.backup_inspect(address, backup4)['Name'] # start no-frontend volume # start dr volume (no frontend) dr_address = grpc_dr_controller.address start_no_frontend_volume(grpc_engine_manager, grpc_dr_controller, grpc_dr_replica1, grpc_dr_replica2) cmd.backup_restore(dr_address, backup0) wait_for_restore_completion(dr_address, backup0) verify_no_frontend_data(grpc_engine_manager, 0, snap0_data, grpc_dr_controller) # mock restore crash/error delta_file1 = "volume-delta-" + backup0_name + ".img" if "vfs" in backup_target: command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME] backup_volume_path = subprocess.check_output(command).strip() command = ["find", backup_volume_path, "-name", "*blk"] blocks = subprocess.check_output(command).split() assert len(blocks) != 0 for blk in blocks: command = ["mv", blk, blk + ".tmp"] subprocess.check_output(command).strip() # should fail is_failed = False cmd.restore_inc(dr_address, backup1, backup0_name) for i in range(RETRY_COUNTS): rs = cmd.restore_status(dr_address) for status in rs.values(): if status['backupURL'] != backup1: break if 'error' in status.keys(): if status['error'] != "": assert 'no such file or directory' in \ status['error'] is_failed = True if is_failed: break time.sleep(RETRY_INTERVAL) assert is_failed assert path.exists(FIXED_REPLICA_PATH1 + delta_file1) assert path.exists(FIXED_REPLICA_PATH2 + delta_file1) for blk in blocks: command = ["mv", blk + ".tmp", blk] subprocess.check_output(command) data1 = \ snap0_data[0:offset1] + snap1_data + \ snap0_data[offset1+length1:] # race condition: last restoration has failed # but `isRestoring` hasn't been cleanup for i in range(RETRY_COUNTS): try: restore_incrementally(dr_address, backup1, backup0_name) break except subprocess.CalledProcessError as e: if "already in progress" not in e.output: time.sleep(RETRY_INTERVAL) else: raise e verify_no_frontend_data(grpc_engine_manager, 0, data1, grpc_dr_controller) assert not path.exists(FIXED_REPLICA_PATH1 + delta_file1) assert not path.exists(FIXED_REPLICA_PATH2 + delta_file1) status = cmd.restore_status(dr_address) compare_last_restored_with_backup(status, backup1_name) data2 = \ data1[0:offset2] + snap2_data + \ zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data restore_incrementally(dr_address, backup2, backup1_name) verify_no_frontend_data(grpc_engine_manager, 0, data2, grpc_dr_controller) delta_file2 = "volume-delta-" + backup1_name + ".img" assert not path.exists(FIXED_REPLICA_PATH1 + delta_file2) assert not path.exists(FIXED_REPLICA_PATH2 + delta_file2) status = cmd.restore_status(dr_address) compare_last_restored_with_backup(status, backup2_name) # mock race condition with pytest.raises(subprocess.CalledProcessError) as e: restore_incrementally(dr_address, backup1, backup0_name) assert "doesn't match lastRestored" in e data3 = zero_string * length3 + data2[length3:length2] restore_incrementally(dr_address, backup3, backup2_name) verify_no_frontend_data(grpc_engine_manager, 0, data3, grpc_dr_controller) delta_file3 = "volume-delta-" + backup3_name + ".img" assert not path.exists(FIXED_REPLICA_PATH1 + delta_file3) assert not path.exists(FIXED_REPLICA_PATH2 + delta_file3) status = cmd.restore_status(dr_address) compare_last_restored_with_backup(status, backup3_name) # mock corner case: invalid last-restored backup rm_backups(address, ENGINE_NAME, [backup3]) # actually it is full restoration restore_incrementally(dr_address, backup4, backup3_name) verify_no_frontend_data(grpc_engine_manager, 0, snap4_data, grpc_dr_controller) status = cmd.restore_status(dr_address) compare_last_restored_with_backup(status, backup4_name) if "vfs" in backup_target: command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME] backup_volume_path = subprocess.check_output(command).strip() command = ["find", backup_volume_path, "-name", "*tempoary"] tmp_files = subprocess.check_output(command).split() assert len(tmp_files) == 0 cleanup_no_frontend_volume(grpc_engine_manager, grpc_dr_controller, grpc_dr_replica1, grpc_dr_replica2) rm_backups(address, ENGINE_NAME, [backup0, backup1, backup2, backup4]) cmd.sync_agent_server_reset(address) cleanup_controller(grpc_controller) cleanup_replica(grpc_replica1) cleanup_replica(grpc_replica2)
def restore_inc_test( controller, replica1, replica2, # NOQA sb_controller, sb_replica1, sb_replica2, backup_target): # NOQA launcher.start_engine_frontend(FRONTEND_TGT_BLOCKDEV, url=LAUNCHER) dev = common.get_dev(replica1, replica2, controller) zero_string = b'\x00'.decode('utf-8') # backup0: 256 random data in 1st block length0 = 256 snap0_data = common.random_string(length0) verify_data(dev, 0, snap0_data) verify_data(dev, BLOCK_SIZE, snap0_data) snap0 = cmd.snapshot_create() backup0 = create_backup(backup_target, snap0) backup0_name = cmd.backup_inspect(backup0)['Name'] # backup1: 32 random data + 32 zero data + 192 random data in 1st block length1 = 32 offset1 = 32 snap1_data = zero_string * length1 verify_data(dev, offset1, snap1_data) snap1 = cmd.snapshot_create() backup1 = create_backup(backup_target, snap1) backup1_name = cmd.backup_inspect(backup1)['Name'] # backup2: 32 random data + 256 random data in 1st block, # 256 random data in 2nd block length2 = 256 offset2 = 32 snap2_data = common.random_string(length2) verify_data(dev, offset2, snap2_data) verify_data(dev, BLOCK_SIZE, snap2_data) snap2 = cmd.snapshot_create() backup2 = create_backup(backup_target, snap2) backup2_name = cmd.backup_inspect(backup2)['Name'] # backup3: 64 zero data + 192 random data in 1st block length3 = 64 offset3 = 0 verify_data(dev, offset3, zero_string * length3) verify_data(dev, length2, zero_string * offset2) verify_data(dev, BLOCK_SIZE, zero_string * length2) snap3 = cmd.snapshot_create() backup3 = create_backup(backup_target, snap3) backup3_name = cmd.backup_inspect(backup3)['Name'] # backup4: 256 random data in 1st block length4 = 256 offset4 = 0 snap4_data = common.random_string(length4) verify_data(dev, offset4, snap4_data) snap4 = cmd.snapshot_create() backup4 = create_backup(backup_target, snap4) backup4_name = cmd.backup_inspect(backup4)['Name'] common.cleanup_replica(replica1) common.cleanup_replica(replica2) common.cleanup_controller(controller) launcher.shutdown_engine_frontend(url=LAUNCHER) # start no-frontend volume # start standby volume (no frontend) start_no_frontend_volume(sb_controller, sb_replica1, sb_replica2) restore_for_no_frontend_volume(backup0, sb_controller) verify_no_frontend_data(0, snap0_data, sb_controller) # mock restore crash/error delta_file1 = "volume-delta-" + backup0_name + ".img" if "vfs" in backup_target: command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME] backup_volume_path = subprocess.check_output(command).strip() command = ["find", backup_volume_path, "-name", "*blk"] blocks = subprocess.check_output(command).split() assert len(blocks) != 0 for blk in blocks: command = ["mv", blk, blk + ".tmp"] subprocess.check_output(command).strip() with pytest.raises(subprocess.CalledProcessError): cmd.restore_inc(backup1, backup0_name, CONTROLLER_NO_FRONTEND) assert path.exists(STANDBY_REPLICA1_PATH + delta_file1) assert path.exists(STANDBY_REPLICA2_PATH + delta_file1) for blk in blocks: command = ["mv", blk + ".tmp", blk] subprocess.check_output(command) data1 = \ snap0_data[0:offset1] + snap1_data + \ snap0_data[offset1+length1:] cmd.restore_inc(backup1, backup0_name, CONTROLLER_NO_FRONTEND) verify_no_frontend_data(0, data1, sb_controller) assert not path.exists(STANDBY_REPLICA1_PATH + delta_file1) assert not path.exists(STANDBY_REPLICA2_PATH + delta_file1) volume_info = cmd.info(CONTROLLER_NO_FRONTEND) assert volume_info['lastRestored'] == backup1_name data2 = \ data1[0:offset2] + snap2_data + \ zero_string * (BLOCK_SIZE - length2 - offset2) + snap2_data cmd.restore_inc(backup2, backup1_name, CONTROLLER_NO_FRONTEND) verify_no_frontend_data(0, data2, sb_controller) delta_file2 = "volume-delta-" + backup1_name + ".img" assert not path.exists(STANDBY_REPLICA1_PATH + delta_file2) assert not path.exists(STANDBY_REPLICA2_PATH + delta_file2) volume_info = cmd.info(CONTROLLER_NO_FRONTEND) assert volume_info['lastRestored'] == backup2_name # mock race condition with pytest.raises(subprocess.CalledProcessError) as e: cmd.restore_inc(backup1, backup0_name, CONTROLLER_NO_FRONTEND) assert "doesn't match lastRestored" in e data3 = zero_string * length3 + data2[length3:length2] cmd.restore_inc(backup3, backup2_name, CONTROLLER_NO_FRONTEND) verify_no_frontend_data(0, data3, sb_controller) delta_file3 = "volume-delta-" + backup3_name + ".img" assert not path.exists(STANDBY_REPLICA1_PATH + delta_file3) assert not path.exists(STANDBY_REPLICA2_PATH + delta_file3) volume_info = cmd.info(CONTROLLER_NO_FRONTEND) assert volume_info['lastRestored'] == backup3_name # mock corner case: invalid last-restored backup rm_backups([backup3]) # actually it is full restoration cmd.restore_inc(backup4, backup3_name, CONTROLLER_NO_FRONTEND) verify_no_frontend_data(0, snap4_data, sb_controller) volume_info = cmd.info(CONTROLLER_NO_FRONTEND) assert volume_info['lastRestored'] == backup4_name if "vfs" in backup_target: command = ["find", VFS_DIR, "-type", "d", "-name", VOLUME_NAME] backup_volume_path = subprocess.check_output(command).strip() command = ["find", backup_volume_path, "-name", "*tempoary"] tmp_files = subprocess.check_output(command).split() assert len(tmp_files) == 0 cleanup_no_frontend_volume(sb_controller, sb_replica1, sb_replica2) rm_backups([backup0, backup1, backup2, backup4])