def backup_with_backing_file_test(backup_target, # NOQA grpc_backing_controller, # NOQA grpc_backing_replica1, # NOQA grpc_backing_replica2): # NOQA address = grpc_backing_controller.address dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2, grpc_backing_controller) offset = 0 length = 256 snap0 = cmd.snapshot_create(address) before = read_dev(dev, offset, length) assert before != "" snap0_checksum = checksum_dev(dev) exists = read_from_backing_file(offset, length) assert before == exists backup0_info = create_backup(address, snap0, backup_target) assert backup0_info["VolumeName"] == VOLUME_BACKING_NAME backup_test(dev, address, VOLUME_BACKING_NAME, ENGINE_BACKING_NAME, backup_target) restore_with_frontend(address, ENGINE_BACKING_NAME, backup0_info["URL"]) after = read_dev(dev, offset, length) assert before == after c = checksum_dev(dev) assert c == snap0_checksum rm_backups(address, ENGINE_BACKING_NAME, [backup0_info["URL"]])
def test_snapshot_revert_with_backing_file(grpc_backing_controller, # NOQA grpc_backing_replica1, # NOQA grpc_backing_replica2): # NOQA address = grpc_backing_controller.address dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2, grpc_backing_controller) offset = 0 length = 256 snap0 = cmd.snapshot_create(address) before = read_dev(dev, offset, length) assert before != "" info = cmd.snapshot_info(address) assert len(info) == 2 assert VOLUME_HEAD in info assert snap0 in info exists = read_from_backing_file(offset, length) assert before == exists snapshot_revert_test(dev, address, ENGINE_BACKING_NAME) snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0) after = read_dev(dev, offset, length) assert before == after
def backup_test( dev, address, # NOQA volume_name, engine_name, backup_target): offset = 0 length = 128 snap1_data = random_string(length) verify_data(dev, offset, snap1_data) snap1_checksum = checksum_dev(dev) snap1 = cmd.snapshot_create(address) backup1_info = create_backup(address, snap1, backup_target) assert backup1_info["VolumeName"] == volume_name assert backup1_info["Size"] == BLOCK_SIZE_STR snap2_data = random_string(length) verify_data(dev, offset, snap2_data) snap2_checksum = checksum_dev(dev) snap2 = cmd.snapshot_create(address) backup2_info = create_backup(address, snap2, backup_target) assert backup2_info["VolumeName"] == volume_name assert backup2_info["Size"] == BLOCK_SIZE_STR snap3_data = random_string(length) verify_data(dev, offset, snap3_data) snap3_checksum = checksum_dev(dev) snap3 = cmd.snapshot_create(address) backup3_info = create_backup(address, snap3, backup_target) assert backup3_info["VolumeName"] == volume_name assert backup3_info["Size"] == BLOCK_SIZE_STR restore_with_frontend(address, engine_name, backup3_info["URL"]) readed = read_dev(dev, offset, length) assert readed == snap3_data c = checksum_dev(dev) assert c == snap3_checksum rm_backups(address, engine_name, [backup3_info["URL"]]) restore_with_frontend(address, engine_name, backup1_info["URL"]) readed = read_dev(dev, offset, length) assert readed == snap1_data c = checksum_dev(dev) assert c == snap1_checksum rm_backups(address, engine_name, [backup1_info["URL"]]) restore_with_frontend(address, engine_name, backup2_info["URL"]) readed = read_dev(dev, offset, length) assert readed == snap2_data c = checksum_dev(dev) assert c == snap2_checksum rm_backups(address, engine_name, [backup2_info["URL"]])
def backup_hole_with_backing_file_test(backup_target, # NOQA grpc_backing_controller, # NOQA grpc_backing_replica1, # NOQA grpc_backing_replica2): # NOQA address = grpc_backing_controller.address dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2, grpc_backing_controller) offset1 = 512 length1 = 256 offset2 = 640 length2 = 256 boundary_offset = 0 boundary_length = 4100 # just pass 4096 into next 4k hole_offset = 2 * 1024 * 1024 hole_length = 1024 snap1_data = random_string(length1) verify_data(dev, offset1, snap1_data) snap1_checksum = checksum_dev(dev) snap1 = cmd.snapshot_create(address) boundary_data_backup1 = read_dev(dev, boundary_offset, boundary_length) hole_data_backup1 = read_dev(dev, hole_offset, hole_length) backup1_info = create_backup(address, snap1, backup_target) snap2_data = random_string(length2) verify_data(dev, offset2, snap2_data) snap2_checksum = checksum_dev(dev) snap2 = cmd.snapshot_create(address) boundary_data_backup2 = read_dev(dev, boundary_offset, boundary_length) hole_data_backup2 = read_dev(dev, hole_offset, hole_length) backup2_info = create_backup(address, snap2, backup_target) restore_with_frontend(address, ENGINE_BACKING_NAME, backup1_info["URL"]) readed = read_dev(dev, boundary_offset, boundary_length) assert readed == boundary_data_backup1 readed = read_dev(dev, hole_offset, hole_length) assert readed == hole_data_backup1 c = checksum_dev(dev) assert c == snap1_checksum restore_with_frontend(address, ENGINE_BACKING_NAME, backup2_info["URL"]) readed = read_dev(dev, boundary_offset, boundary_length) assert readed == boundary_data_backup2 readed = read_dev(dev, hole_offset, hole_length) assert readed == hole_data_backup2 c = checksum_dev(dev) assert c == snap2_checksum
def snapshot_tree_verify_backup_node( grpc_controller, grpc_replica1, grpc_replica2, address, engine_name, offset, length, backup, data, name): # NOQA reset_volume(grpc_controller, grpc_replica1, grpc_replica2) dev = get_blockdev(grpc_controller.volume_get().name) restore_with_frontend(address, engine_name, backup[name]) readed = read_dev(dev, offset, length) assert readed == data[name]
def test_backup_lock( grpc_replica1, grpc_replica2, # NOQA grpc_controller, backup_targets): # NOQA """ Test backup locks Context: The idea is to implement a locking mechanism that utilizes the backupstore, to prevent the following dangerous cases of concurrent operations. - prevent backup deletion during backup restoration - prevent backup deletion while a backup is in progress - prevent backup creation during backup deletion - prevent backup restoration during backup deletion Steps: 1. Create a volume(1) and attach to the current node 2. create a backup(1) of volume(1) 3. verify backup(1) creation completed 4. write some data to volume(1) 5. create an active lock of type Delete 6. create a backup(2) of volume(1) 7. verify backup(2) creation timed out 8. delete active lock of type Delete 9. create an active lock of type Delete 10. restore backup(1) 11. verify backup(1) restore timed out 12. delete active lock of type Delete 13. restore backup(1) 14. verify backup(1) restore completed 15. create an active lock of type Restore 16. delete backup(1) 17. verify backup(1) deletion timed out 18. delete active lock of type Restore 19. delete backup(1) 20. verify backup(1) deletion completed 21. cleanup """ for backup_target in backup_targets: dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller) # create a regular backup address = grpc_controller.address offset = 0 length = 128 snap1_data = random_string(length) verify_data(dev, offset, snap1_data) snap1_checksum = checksum_dev(dev) snap1 = cmd.snapshot_create(address) # create a backup to create the volume info = create_backup(address, snap1, backup_target) assert info["VolumeName"] == VOLUME_NAME assert info["Size"] == BLOCK_SIZE_STR assert snap1 in info["SnapshotName"] # backup should error out with timeout # because of delete lock create_delete_lock(True) with pytest.raises(subprocess.CalledProcessError): create_backup(address, snap1, backup_target) remove_lock_file(DELETE_LOCK) # restore should error out with timeout # because of delete lock create_delete_lock(True) with pytest.raises(subprocess.CalledProcessError): restore_with_frontend(address, ENGINE_NAME, info["URL"]) remove_lock_file(DELETE_LOCK) # restore should succeed now, that there is no active delete lock restore_with_frontend(address, ENGINE_NAME, info["URL"]) readed = read_dev(dev, offset, length) assert readed == snap1_data c = checksum_dev(dev) assert c == snap1_checksum # delete should error out with timeout # because of restore lock create_restore_lock(True) with pytest.raises(subprocess.CalledProcessError): rm_backups(address, ENGINE_NAME, [info["URL"]]) remove_lock_file(RESTORE_LOCK) # delete should succeed now, that there is no active restore lock rm_backups(address, ENGINE_NAME, [info["URL"]]) # cleanup volume 1 cmd.sync_agent_server_reset(address) cleanup_controller(grpc_controller) cleanup_replica(grpc_replica1) cleanup_replica(grpc_replica2)
def test_backup_incremental_logic(grpc_replica1, grpc_replica2, grpc_controller, backup_targets): # NOQA for backup_target in backup_targets: dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller) address = grpc_controller.address volume_name = VOLUME_NAME engine_name = ENGINE_NAME offset = 0 length = 128 # initial backup snap1_data = random_string(length) verify_data(dev, offset, snap1_data) snap1_checksum = checksum_dev(dev) snap1 = cmd.snapshot_create(address) backup1_info = create_backup(address, snap1, backup_target) assert backup1_info["IsIncremental"] is False # delta backup on top of initial backup snap2_data = random_string(int(length / 2)) verify_data(dev, offset, snap2_data) snap2 = cmd.snapshot_create(address) backup2_info = create_backup(address, snap2, backup_target) assert backup2_info["IsIncremental"] is True # delete the volume cmd.sync_agent_server_reset(address) grpc_controller = cleanup_controller(grpc_controller) grpc_replica1 = cleanup_replica(grpc_replica1) grpc_replica2 = cleanup_replica(grpc_replica2) # recreate the volume dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller, clean_backup_dir=False) # empty initial backup after volume recreation snap3 = cmd.snapshot_create(address) backup3_info = create_backup(address, snap3, backup_target) assert backup3_info["VolumeName"] == volume_name assert backup3_info["Size"] == '0' assert backup3_info["IsIncremental"] is False # write half of snap1 onto head snap4_data = snap1_data[:int(length / 2)] assert len(snap4_data) == int(length / 2) verify_data(dev, offset, snap4_data) snap4_checksum = checksum_dev(dev) assert snap4_checksum != snap1_checksum snap4 = cmd.snapshot_create(address) backup4_info = create_backup(address, snap4, backup_target) assert backup4_info["IsIncremental"] is True # restore initial backup reset_volume(grpc_controller, grpc_replica1, grpc_replica2) dev = get_blockdev(volume_name) restore_with_frontend(address, engine_name, backup1_info["URL"]) assert read_dev(dev, offset, length) == snap1_data assert checksum_dev(dev) == snap1_checksum # restore final backup (half of snap1) reset_volume(grpc_controller, grpc_replica1, grpc_replica2) dev = get_blockdev(volume_name) restore_with_frontend(address, engine_name, backup4_info["URL"]) assert checksum_dev(dev) == snap4_checksum assert snap4_checksum != snap1_checksum data = read_dev(dev, offset, length) assert data[:int(length / 2)] == snap4_data assert data[int(length / 2):] == '\x00' * int(length / 2) rm_backups(address, engine_name, [ backup1_info["URL"], backup2_info["URL"], backup3_info["URL"], backup4_info["URL"] ]) cmd.sync_agent_server_reset(address) cleanup_controller(grpc_controller) cleanup_replica(grpc_replica1) cleanup_replica(grpc_replica2)
def test_backup_S3_latest_unavailable( grpc_replica1, grpc_replica2, # NOQA grpc_controller, backup_targets): # NOQA for backup_target in backup_targets: if "s3://" not in backup_target: continue dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller) address = grpc_controller.address volume_name = VOLUME_NAME engine_name = ENGINE_NAME offset = 0 length = 128 # initial backup snap1_data = random_string(length) verify_data(dev, offset, snap1_data) snap1_checksum = checksum_dev(dev) snap1 = cmd.snapshot_create(address) backup1_info = create_backup(address, snap1, backup_target) # backup to be unavailable snap2_data = random_string(length) verify_data(dev, offset, snap2_data) snap2 = cmd.snapshot_create(address) backup2_info = create_backup(address, snap2, backup_target) # the gc after the restore will clean up the missing backup cfg = findfile(BACKUP_DIR, "backup_" + backup2_info["Name"] + ".cfg") os.remove(cfg) # final full backup after unavailable backup snap3_data = random_string(length) verify_data(dev, offset, snap3_data) snap3_checksum = checksum_dev(dev) snap3 = cmd.snapshot_create(address) backup3_info = create_backup(address, snap3, backup_target) assert backup3_info["VolumeName"] == volume_name assert backup3_info["Size"] == BLOCK_SIZE_STR # write some stuff on head head_data = random_string(length) verify_data(dev, offset, head_data) # test restore of the initial backup reset_volume(grpc_controller, grpc_replica1, grpc_replica2) dev = get_blockdev(volume_name) restore_with_frontend(address, engine_name, backup1_info["URL"]) readed = read_dev(dev, offset, length) assert readed == snap1_data c = checksum_dev(dev) assert c == snap1_checksum # test a restore for the final backup reset_volume(grpc_controller, grpc_replica1, grpc_replica2) dev = get_blockdev(volume_name) restore_with_frontend(address, engine_name, backup3_info["URL"]) readed = read_dev(dev, offset, length) assert readed == snap3_data c = checksum_dev(dev) assert c == snap3_checksum rm_backups(address, engine_name, [backup1_info["URL"], backup3_info["URL"]]) cmd.sync_agent_server_reset(address) cleanup_controller(grpc_controller) cleanup_replica(grpc_replica1) cleanup_replica(grpc_replica2)
def snapshot_tree_verify_node(dev, address, engine_name, offset, length, snap, data, name): snapshot_revert_with_frontend(address, engine_name, snap[name]) readed = read_dev(dev, offset, length) assert readed == data[name]
def restore_to_file_with_backing_file_test( backup_target, # NOQA grpc_backing_controller, # NOQA grpc_backing_replica1, # NOQA grpc_backing_replica2): # NOQA address = grpc_backing_controller.address backing_dev = get_dev(grpc_backing_replica1, grpc_backing_replica2, grpc_backing_controller) length0 = 4 * 1024 length1 = 256 length2 = 128 offset0 = 0 offset1 = length1 + offset0 offset2 = length2 + offset0 output_raw_path = file(OUTPUT_FILE_RAW) output_qcow2_path = file(OUTPUT_FILE_QCOW2) # create 1 empty snapshot. # data in output image == data in backing check_backing() check_empty_volume(backing_dev) snap0 = cmd.snapshot_create(address) backup = create_backup(address, snap0, backup_target)["URL"] volume_data = read_dev(backing_dev, offset0, length0) backing_data = read_from_backing_file(offset0, length0) dev_checksum = checksum_dev(backing_dev) assert volume_data != "" assert volume_data == backing_data cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_raw_path, IMAGE_FORMAT_RAW) output0_raw = read_file(output_raw_path, offset0, length0) output0_checksum = checksum_data( read_file(output_raw_path, 0, SIZE).encode('utf-8')) assert output0_raw == backing_data assert output0_checksum == dev_checksum os.remove(output_raw_path) assert not os.path.exists(output_raw_path) cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_qcow2_path, IMAGE_FORMAT_QCOW2) output0_qcow2 = read_qcow2_file_without_backing_file( output_qcow2_path, offset0, length0) output0_checksum = checksum_data( read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE)) assert output0_qcow2.decode('utf-8') == backing_data assert output0_qcow2.decode('utf-8') == volume_data assert output0_checksum == dev_checksum os.remove(output_qcow2_path) assert not os.path.exists(output_qcow2_path) rm_backups(address, ENGINE_BACKING_NAME, [backup]) # create 1 snapshot with 256B data. # output = snap1(offset0, length1) + backing(offset1, ...) snap1_data = random_string(length1) verify_data(backing_dev, offset0, snap1_data) snap1 = cmd.snapshot_create(address) backup = create_backup(address, snap1, backup_target)["URL"] volume_data = read_dev(backing_dev, offset0, length0) backing_data = read_from_backing_file(offset1, length0 - offset1) dev_checksum = checksum_dev(backing_dev) cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_raw_path, IMAGE_FORMAT_RAW) output1_raw_snap1 = read_file(output_raw_path, offset0, length1) output1_raw_backing = read_file(output_raw_path, offset1, length0 - offset1) output1_checksum = checksum_data( read_file(output_raw_path, 0, SIZE).encode('utf-8')) assert output1_raw_snap1 == snap1_data assert output1_raw_backing == backing_data assert output1_raw_snap1 + output1_raw_backing == volume_data assert output1_checksum == dev_checksum os.remove(output_raw_path) assert not os.path.exists(output_raw_path) cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_qcow2_path, IMAGE_FORMAT_QCOW2) output1_qcow2_snap1 = read_qcow2_file_without_backing_file( output_qcow2_path, offset0, length1) output1_qcow2_backing = read_qcow2_file_without_backing_file( output_qcow2_path, offset1, length0 - offset1) output1_checksum = checksum_data( read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE)) assert output1_qcow2_snap1.decode('utf-8') == snap1_data assert output1_qcow2_backing.decode('utf-8') == backing_data assert output1_qcow2_snap1.decode('utf-8') + \ output1_qcow2_backing.decode('utf-8') == volume_data assert output1_checksum == dev_checksum os.remove(output_qcow2_path) assert not os.path.exists(output_qcow2_path) snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0) rm_snaps(address, [snap1]) rm_backups(address, ENGINE_BACKING_NAME, [backup]) check_backing() check_empty_volume(backing_dev) # create 2 snapshots with 256B data and 128B data # output = snap2(offset0, length1 - length2) + # snap1(offset2, length2) + backing(offset2, ...) snap1_data = random_string(length1) verify_data(backing_dev, offset0, snap1_data) snap1 = cmd.snapshot_create(address) snap2_data = random_string(length2) verify_data(backing_dev, offset0, snap2_data) snap2 = cmd.snapshot_create(address) backup = create_backup(address, snap2, backup_target)["URL"] volume_data = read_dev(backing_dev, offset0, length0) backing_data = read_from_backing_file(offset1, length0 - offset1) dev_checksum = checksum_dev(backing_dev) cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_raw_path, IMAGE_FORMAT_RAW) output2_raw_snap2 = read_file(output_raw_path, offset0, length2) output2_raw_snap1 = read_file(output_raw_path, offset2, length1 - length2) output2_raw_backing = read_file(output_raw_path, offset1, length0 - offset1) output2_checksum = checksum_data( read_file(output_raw_path, 0, SIZE).encode('utf-8')) assert output2_raw_snap2 == snap2_data assert output2_raw_snap1 == snap1_data[offset2:length1] assert output2_raw_backing == backing_data assert \ volume_data == \ output2_raw_snap2 + output2_raw_snap1 + output2_raw_backing assert output2_checksum == dev_checksum os.remove(output_raw_path) assert not os.path.exists(output_raw_path) cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_qcow2_path, IMAGE_FORMAT_QCOW2) output2_qcow2_snap2 = read_qcow2_file_without_backing_file( output_qcow2_path, offset0, length2) output2_qcow2_snap1 = read_qcow2_file_without_backing_file( output_qcow2_path, offset2, length1 - length2) output2_qcow2_backing = read_qcow2_file_without_backing_file( output_qcow2_path, offset1, length0 - offset1) output2_checksum = checksum_data( read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE)) assert output2_qcow2_snap2.decode('utf-8') == snap2_data assert output2_qcow2_snap1.decode('utf-8') == snap1_data[offset2:length1] assert output2_qcow2_backing.decode('utf-8') == backing_data assert \ volume_data == \ output2_qcow2_snap2.decode('utf-8') + \ output2_qcow2_snap1.decode('utf-8') + \ output1_qcow2_backing.decode('utf-8') assert output2_checksum == dev_checksum os.remove(output_qcow2_path) assert not os.path.exists(output_qcow2_path) snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0) rm_snaps(address, [snap1, snap2]) rm_backups(address, ENGINE_BACKING_NAME, [backup]) check_backing() check_empty_volume(backing_dev)
def check_empty_volume(dev, offset=0, length=4 * 1024): backing_data = read_from_backing_file(offset, length) empty_volume_data = read_dev(dev, offset, length) assert backing_data == empty_volume_data
def snapshot_tree_verify_backup_node(dev, address, engine_name, offset, length, backup, data, name): restore_with_frontend(address, engine_name, backup[name]) readed = read_dev(dev, offset, length) assert readed == data[name]
def backup_hole_with_backing_file_test( backup_target, # NOQA grpc_backing_controller, # NOQA grpc_backing_replica1, # NOQA grpc_backing_replica2): # NOQA address = grpc_backing_controller.address dev = get_dev(grpc_backing_replica1, grpc_backing_replica2, grpc_backing_controller) volume_name = grpc_backing_controller.volume_get().name assert volume_name == VOLUME_BACKING_NAME offset1 = 512 length1 = 256 offset2 = 640 length2 = 256 boundary_offset = 0 boundary_length = 4100 # just pass 4096 into next 4k hole_offset = 2 * 1024 * 1024 hole_length = 1024 snap1_data = random_string(length1) verify_data(dev, offset1, snap1_data) snap1_checksum = checksum_dev(dev) snap1 = cmd.snapshot_create(address) boundary_data_backup1 = read_dev(dev, boundary_offset, boundary_length) hole_data_backup1 = read_dev(dev, hole_offset, hole_length) backup1_info = create_backup(address, snap1, backup_target, backing_image_name=BACKING_IMAGE_NAME, backing_image_url=BACKING_IMAGE_URL) snap2_data = random_string(length2) verify_data(dev, offset2, snap2_data) snap2_checksum = checksum_dev(dev) snap2 = cmd.snapshot_create(address) boundary_data_backup2 = read_dev(dev, boundary_offset, boundary_length) hole_data_backup2 = read_dev(dev, hole_offset, hole_length) backup2_info = create_backup(address, snap2, backup_target, backing_image_name=BACKING_IMAGE_NAME, backing_image_url=BACKING_IMAGE_URL) assert backup2_info["VolumeBackingImageName"] == BACKING_IMAGE_NAME assert backup2_info["VolumeBackingImageURL"] == BACKING_IMAGE_URL reset_volume(grpc_backing_controller, grpc_backing_replica1, grpc_backing_replica2) dev = get_blockdev(volume_name) restore_with_frontend(address, ENGINE_BACKING_NAME, backup1_info["URL"]) readed = read_dev(dev, boundary_offset, boundary_length) assert readed == boundary_data_backup1 readed = read_dev(dev, hole_offset, hole_length) assert readed == hole_data_backup1 c = checksum_dev(dev) assert c == snap1_checksum reset_volume(grpc_backing_controller, grpc_backing_replica1, grpc_backing_replica2) dev = get_blockdev(volume_name) restore_with_frontend(address, ENGINE_BACKING_NAME, backup2_info["URL"]) readed = read_dev(dev, boundary_offset, boundary_length) assert readed == boundary_data_backup2 readed = read_dev(dev, hole_offset, hole_length) assert readed == hole_data_backup2 c = checksum_dev(dev) assert c == snap2_checksum