def test_snapshot_revert_with_backing_file(grpc_backing_controller, # NOQA grpc_backing_replica1, # NOQA grpc_backing_replica2): # NOQA address = grpc_backing_controller.address dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2, grpc_backing_controller) offset = 0 length = 256 snap0 = cmd.snapshot_create(address) before = read_dev(dev, offset, length) assert before != "" info = cmd.snapshot_info(address) assert len(info) == 2 assert VOLUME_HEAD in info assert snap0 in info exists = read_from_backing_file(offset, length) assert before == exists snapshot_revert_test(dev, address, ENGINE_BACKING_NAME) snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0) after = read_dev(dev, offset, length) assert before == after
def snapshot_revert_test(dev, address, engine_name): # NOQA existings = {} snap1 = Snapshot(dev, generate_random_data(existings), address) snap2 = Snapshot(dev, generate_random_data(existings), address) snap3 = Snapshot(dev, generate_random_data(existings), address) snapList = cmd.snapshot_ls(address) assert snap1.name in snapList assert snap2.name in snapList assert snap3.name in snapList snapshot_revert_with_frontend(address, engine_name, snap2.name) snap3.refute_data() snap2.verify_checksum() snap1.verify_data() snapshot_revert_with_frontend(address, engine_name, snap1.name) snap3.refute_data() snap2.refute_data() snap1.verify_checksum()
def snapshot_tree_build(dev, address, engine_name, offset, length, strict=True): # snap["0a"] -> snap["0b"] -> snap["0c"] # |-> snap["1a"] -> snap["1b"] -> snap["1c"] # \-> snap["2a"] -> snap["2b"] -> snap["2c"] # \-> snap["3a"] -> snap["3b"] -> snap["3c"] -> head snap = {} data = {} snapshot_tree_create_node(dev, address, offset, length, snap, data, "0a") snapshot_tree_create_node(dev, address, offset, length, snap, data, "0b") snapshot_tree_create_node(dev, address, offset, length, snap, data, "0c") snapshot_revert_with_frontend(address, engine_name, snap["0b"]) snapshot_tree_create_node(dev, address, offset, length, snap, data, "1a") snapshot_tree_create_node(dev, address, offset, length, snap, data, "1b") snapshot_tree_create_node(dev, address, offset, length, snap, data, "1c") snapshot_revert_with_frontend(address, engine_name, snap["0b"]) snapshot_tree_create_node(dev, address, offset, length, snap, data, "2a") snapshot_tree_create_node(dev, address, offset, length, snap, data, "2b") snapshot_tree_create_node(dev, address, offset, length, snap, data, "2c") snapshot_revert_with_frontend(address, engine_name, snap["2a"]) snapshot_tree_create_node(dev, address, offset, length, snap, data, "3a") snapshot_tree_create_node(dev, address, offset, length, snap, data, "3b") snapshot_tree_create_node(dev, address, offset, length, snap, data, "3c") snapshot_tree_verify(dev, address, engine_name, offset, length, snap, data, strict) return snap, data
def test_snapshot_rm_basic(grpc_controller, # NOQA grpc_replica1, grpc_replica2): # NOQA address = grpc_controller.address dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller) existings = {} snap1 = Snapshot(dev, generate_random_data(existings), address) snap2 = Snapshot(dev, generate_random_data(existings), address) snap3 = Snapshot(dev, generate_random_data(existings), address) info = cmd.snapshot_info(address) assert len(info) == 4 assert VOLUME_HEAD in info assert snap1.name in info assert snap2.name in info assert snap3.name in info cmd.snapshot_rm(address, snap2.name) cmd.snapshot_purge(address) wait_for_purge_completion(address) info = cmd.snapshot_info(address) assert len(info) == 3 assert snap1.name in info assert snap3.name in info snap3.verify_checksum() snap2.verify_data() snap1.verify_data() snapshot_revert_with_frontend(address, ENGINE_NAME, snap1.name) snap3.refute_data() snap2.refute_data() snap1.verify_checksum()
def snapshot_tree_verify_node(dev, address, engine_name, offset, length, snap, data, name): snapshot_revert_with_frontend(address, engine_name, snap[name]) readed = read_dev(dev, offset, length) assert readed == data[name]
def volume_expansion_with_snapshots_test(dev, grpc_controller, # NOQA volume_name, engine_name, original_data): # the default size is 4MB, will expand it to 8MB address = grpc_controller.address zero_char = b'\x00'.decode('utf-8') # write the data to the original part then do expansion data1_len = random_length(PAGE_SIZE) data1 = Data(random.randrange(0, SIZE-2*PAGE_SIZE, PAGE_SIZE), data1_len, random_string(data1_len)) snap1 = Snapshot(dev, data1, address) expand_volume_with_frontend(grpc_controller, EXPANDED_SIZE) wait_and_check_volume_expansion( grpc_controller, EXPANDED_SIZE) snap1.verify_data() assert \ dev.readat(0, SIZE) == \ original_data[0:data1.offset] + data1.content + \ original_data[data1.offset+data1.length:] assert dev.readat(SIZE, SIZE) == zero_char*SIZE # write the data to both the original part and the expanded part data2_len = random_length(PAGE_SIZE) data2 = Data(SIZE-PAGE_SIZE, data2_len, random_string(data2_len)) snap2 = Snapshot(dev, data2, address) data3_len = random_length(PAGE_SIZE) data3 = Data(random.randrange(SIZE, EXPANDED_SIZE-PAGE_SIZE, PAGE_SIZE), data3_len, random_string(data3_len)) snap3 = Snapshot(dev, data3, address) snap1.verify_data() snap2.verify_data() snap3.verify_data() assert \ dev.readat(SIZE, SIZE) == zero_char*(data3.offset-SIZE) + \ data3.content + zero_char*(EXPANDED_SIZE-data3.offset-data3.length) data4_len = random_length(PAGE_SIZE) data4 = Data(data1.offset, data4_len, random_string(data4_len)) snap4 = Snapshot(dev, data4, address) snap4.verify_data() # revert to snap1 then see if we can still r/w the existing data # and expanded part snapshot_revert_with_frontend(address, engine_name, snap1.name) assert \ dev.readat(0, SIZE) == \ original_data[0:data1.offset] + data1.content + \ original_data[data1.offset+data1.length:] assert dev.readat(SIZE, SIZE) == zero_char*SIZE data5_len = random_length(PAGE_SIZE) data5 = Data(random.randrange(SIZE, EXPANDED_SIZE-PAGE_SIZE, PAGE_SIZE), data5_len, random_string(data5_len)) snap5 = Snapshot(dev, data5, address) snap5.verify_data() assert \ dev.readat(SIZE, SIZE) == zero_char*(data5.offset-SIZE) + \ data5.content + zero_char*(EXPANDED_SIZE-data5.offset-data5.length) # delete and purge the snap1. it will coalesce with the larger snap2 cmd.snapshot_rm(address, snap1.name) cmd.snapshot_purge(address) wait_for_purge_completion(address) assert \ dev.readat(0, SIZE) == \ original_data[0:data1.offset] + data1.content + \ original_data[data1.offset+data1.length:] assert \ dev.readat(SIZE, SIZE) == zero_char*(data5.offset-SIZE) + \ data5.content + zero_char*(EXPANDED_SIZE-data5.offset-data5.length)
def test_snapshot_tree_basic(grpc_controller, # NOQA grpc_replica1, grpc_replica2): # NOQA address = grpc_controller.address dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller) offset = 0 length = 128 snap, data = snapshot_tree_build(dev, address, ENGINE_NAME, offset, length) snapshot_revert_with_frontend(address, ENGINE_NAME, snap["1b"]) cmd.snapshot_rm(address, snap["0a"]) cmd.snapshot_rm(address, snap["0b"]) cmd.snapshot_rm(address, snap["1c"]) cmd.snapshot_rm(address, snap["2a"]) cmd.snapshot_rm(address, snap["2b"]) cmd.snapshot_rm(address, snap["2c"]) cmd.snapshot_rm(address, snap["3a"]) cmd.snapshot_rm(address, snap["3b"]) cmd.snapshot_rm(address, snap["3c"]) cmd.snapshot_purge(address) wait_for_purge_completion(address) # the result should looks like this # snap["0b"](r) -> snap["0c"] # \-> snap["1a"] -> snap["1b"] -> head info = cmd.snapshot_info(address) assert len(info) == 5 assert snap["0b"] in info assert info[snap["0b"]]["parent"] == "" assert len(info[snap["0b"]]["children"]) == 2 assert snap["0c"] in info[snap["0b"]]["children"] assert snap["1a"] in info[snap["0b"]]["children"] assert info[snap["0b"]]["removed"] is True assert snap["0c"] in info assert info[snap["0c"]]["parent"] == snap["0b"] assert not info[snap["0c"]]["children"] assert snap["1a"] in info assert info[snap["1a"]]["parent"] == snap["0b"] assert snap["1b"] in info[snap["1a"]]["children"] assert snap["1b"] in info assert info[snap["1b"]]["parent"] == snap["1a"] assert VOLUME_HEAD in info[snap["1b"]]["children"] assert VOLUME_HEAD in info assert info[VOLUME_HEAD]["parent"] == snap["1b"] snapshot_tree_verify_node(dev, address, ENGINE_NAME, offset, length, snap, data, "0b") snapshot_tree_verify_node(dev, address, ENGINE_NAME, offset, length, snap, data, "0c") snapshot_tree_verify_node(dev, address, ENGINE_NAME, offset, length, snap, data, "1a") snapshot_tree_verify_node(dev, address, ENGINE_NAME, offset, length, snap, data, "1b")
def restore_to_file_with_backing_file_test( backup_target, # NOQA grpc_backing_controller, # NOQA grpc_backing_replica1, # NOQA grpc_backing_replica2): # NOQA address = grpc_backing_controller.address backing_dev = get_dev(grpc_backing_replica1, grpc_backing_replica2, grpc_backing_controller) length0 = 4 * 1024 length1 = 256 length2 = 128 offset0 = 0 offset1 = length1 + offset0 offset2 = length2 + offset0 output_raw_path = file(OUTPUT_FILE_RAW) output_qcow2_path = file(OUTPUT_FILE_QCOW2) # create 1 empty snapshot. # data in output image == data in backing check_backing() check_empty_volume(backing_dev) snap0 = cmd.snapshot_create(address) backup = create_backup(address, snap0, backup_target)["URL"] volume_data = read_dev(backing_dev, offset0, length0) backing_data = read_from_backing_file(offset0, length0) dev_checksum = checksum_dev(backing_dev) assert volume_data != "" assert volume_data == backing_data cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_raw_path, IMAGE_FORMAT_RAW) output0_raw = read_file(output_raw_path, offset0, length0) output0_checksum = checksum_data( read_file(output_raw_path, 0, SIZE).encode('utf-8')) assert output0_raw == backing_data assert output0_checksum == dev_checksum os.remove(output_raw_path) assert not os.path.exists(output_raw_path) cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_qcow2_path, IMAGE_FORMAT_QCOW2) output0_qcow2 = read_qcow2_file_without_backing_file( output_qcow2_path, offset0, length0) output0_checksum = checksum_data( read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE)) assert output0_qcow2.decode('utf-8') == backing_data assert output0_qcow2.decode('utf-8') == volume_data assert output0_checksum == dev_checksum os.remove(output_qcow2_path) assert not os.path.exists(output_qcow2_path) rm_backups(address, ENGINE_BACKING_NAME, [backup]) # create 1 snapshot with 256B data. # output = snap1(offset0, length1) + backing(offset1, ...) snap1_data = random_string(length1) verify_data(backing_dev, offset0, snap1_data) snap1 = cmd.snapshot_create(address) backup = create_backup(address, snap1, backup_target)["URL"] volume_data = read_dev(backing_dev, offset0, length0) backing_data = read_from_backing_file(offset1, length0 - offset1) dev_checksum = checksum_dev(backing_dev) cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_raw_path, IMAGE_FORMAT_RAW) output1_raw_snap1 = read_file(output_raw_path, offset0, length1) output1_raw_backing = read_file(output_raw_path, offset1, length0 - offset1) output1_checksum = checksum_data( read_file(output_raw_path, 0, SIZE).encode('utf-8')) assert output1_raw_snap1 == snap1_data assert output1_raw_backing == backing_data assert output1_raw_snap1 + output1_raw_backing == volume_data assert output1_checksum == dev_checksum os.remove(output_raw_path) assert not os.path.exists(output_raw_path) cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_qcow2_path, IMAGE_FORMAT_QCOW2) output1_qcow2_snap1 = read_qcow2_file_without_backing_file( output_qcow2_path, offset0, length1) output1_qcow2_backing = read_qcow2_file_without_backing_file( output_qcow2_path, offset1, length0 - offset1) output1_checksum = checksum_data( read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE)) assert output1_qcow2_snap1.decode('utf-8') == snap1_data assert output1_qcow2_backing.decode('utf-8') == backing_data assert output1_qcow2_snap1.decode('utf-8') + \ output1_qcow2_backing.decode('utf-8') == volume_data assert output1_checksum == dev_checksum os.remove(output_qcow2_path) assert not os.path.exists(output_qcow2_path) snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0) rm_snaps(address, [snap1]) rm_backups(address, ENGINE_BACKING_NAME, [backup]) check_backing() check_empty_volume(backing_dev) # create 2 snapshots with 256B data and 128B data # output = snap2(offset0, length1 - length2) + # snap1(offset2, length2) + backing(offset2, ...) snap1_data = random_string(length1) verify_data(backing_dev, offset0, snap1_data) snap1 = cmd.snapshot_create(address) snap2_data = random_string(length2) verify_data(backing_dev, offset0, snap2_data) snap2 = cmd.snapshot_create(address) backup = create_backup(address, snap2, backup_target)["URL"] volume_data = read_dev(backing_dev, offset0, length0) backing_data = read_from_backing_file(offset1, length0 - offset1) dev_checksum = checksum_dev(backing_dev) cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_raw_path, IMAGE_FORMAT_RAW) output2_raw_snap2 = read_file(output_raw_path, offset0, length2) output2_raw_snap1 = read_file(output_raw_path, offset2, length1 - length2) output2_raw_backing = read_file(output_raw_path, offset1, length0 - offset1) output2_checksum = checksum_data( read_file(output_raw_path, 0, SIZE).encode('utf-8')) assert output2_raw_snap2 == snap2_data assert output2_raw_snap1 == snap1_data[offset2:length1] assert output2_raw_backing == backing_data assert \ volume_data == \ output2_raw_snap2 + output2_raw_snap1 + output2_raw_backing assert output2_checksum == dev_checksum os.remove(output_raw_path) assert not os.path.exists(output_raw_path) cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW), output_qcow2_path, IMAGE_FORMAT_QCOW2) output2_qcow2_snap2 = read_qcow2_file_without_backing_file( output_qcow2_path, offset0, length2) output2_qcow2_snap1 = read_qcow2_file_without_backing_file( output_qcow2_path, offset2, length1 - length2) output2_qcow2_backing = read_qcow2_file_without_backing_file( output_qcow2_path, offset1, length0 - offset1) output2_checksum = checksum_data( read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE)) assert output2_qcow2_snap2.decode('utf-8') == snap2_data assert output2_qcow2_snap1.decode('utf-8') == snap1_data[offset2:length1] assert output2_qcow2_backing.decode('utf-8') == backing_data assert \ volume_data == \ output2_qcow2_snap2.decode('utf-8') + \ output2_qcow2_snap1.decode('utf-8') + \ output1_qcow2_backing.decode('utf-8') assert output2_checksum == dev_checksum os.remove(output_qcow2_path) assert not os.path.exists(output_qcow2_path) snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0) rm_snaps(address, [snap1, snap2]) rm_backups(address, ENGINE_BACKING_NAME, [backup]) check_backing() check_empty_volume(backing_dev)
def restore_to_file_without_backing_file_test( backup_target, # NOQA grpc_controller, # NOQA grpc_replica1, # NOQA grpc_replica2): # NOQA address = grpc_controller.address dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller) length0 = 256 length1 = 128 offset0 = 0 offset1 = length1 + offset0 output_raw_path = file(OUTPUT_FILE_RAW) output_qcow2_path = file(OUTPUT_FILE_QCOW2) # create 1 empty snapshot for converting to init state. snap0 = cmd.snapshot_create(address) # create 1 snapshot with 256B data. # output = snap2(offset0, length1) snap1_data = random_string(length0) verify_data(dev, offset0, snap1_data) snap1 = cmd.snapshot_create(address) backup = create_backup(address, snap1, backup_target)["URL"] cmd.restore_to_file(address, backup, "", output_raw_path, IMAGE_FORMAT_RAW) output1_raw = read_file(output_raw_path, offset0, length0) assert output1_raw == snap1_data os.remove(output_raw_path) assert not os.path.exists(output_raw_path) cmd.restore_to_file(address, backup, "", output_qcow2_path, IMAGE_FORMAT_QCOW2) output1_qcow2 = read_qcow2_file_without_backing_file( output_qcow2_path, offset0, length0) assert output1_qcow2.decode('utf-8') == snap1_data os.remove(output_qcow2_path) assert not os.path.exists(output_qcow2_path) snapshot_revert_with_frontend(address, ENGINE_NAME, snap0) rm_snaps(address, [snap1]) rm_backups(address, ENGINE_NAME, [backup]) # create 2 snapshots with 256B data and 128B data # output = snap2(offset0, length1 - length2) + # snap1(offset2, length2) snap1_data = random_string(length0) verify_data(dev, offset0, snap1_data) snap1 = cmd.snapshot_create(address) snap2_data = random_string(length1) verify_data(dev, offset0, snap2_data) snap2 = cmd.snapshot_create(address) backup = create_backup(address, snap2, backup_target)["URL"] cmd.restore_to_file(address, backup, "", output_raw_path, IMAGE_FORMAT_RAW) output2_raw_snap2 = read_file(output_raw_path, offset0, length1) output2_raw_snap1 = read_file(output_raw_path, offset1, length0 - length1) assert output2_raw_snap2 == snap2_data assert output2_raw_snap1 == snap1_data[offset1:length0] cmd.restore_to_file(address, backup, "", output_qcow2_path, IMAGE_FORMAT_QCOW2) output2_qcow2_snap2 = read_qcow2_file_without_backing_file( output_qcow2_path, offset0, length1) output2_qcow2_snap1 = read_qcow2_file_without_backing_file( output_qcow2_path, offset1, length0 - length1) assert output2_qcow2_snap2.decode('utf-8') == snap2_data assert output2_qcow2_snap1.decode('utf-8') == snap1_data[offset1:length0] os.remove(output_qcow2_path) assert not os.path.exists(output_qcow2_path) snapshot_revert_with_frontend(address, ENGINE_NAME, snap0) rm_snaps(address, [snap1, snap2]) rm_backups(address, ENGINE_NAME, [backup])
def snapshot_mounted_filesystem_test(volume_name, dev, address, engine_name): # NOQA dev_path = dev.dev mnt_path = "/tmp/mnt-" + volume_name test_file = mnt_path + "/test" length = 128 print("dev_path: " + dev_path + "\n") print("mnt_path: " + mnt_path + "\n") # create & mount a ext4 filesystem on dev nsenter_cmd = [ "nsenter", "--mount=/host/proc/1/ns/mnt", "--net=/host/proc/1/ns/net", "--" ] mount_cmd = nsenter_cmd + ["mount", "--make-shared", dev_path, mnt_path] umount_cmd = nsenter_cmd + ["umount", mnt_path] findmnt_cmd = nsenter_cmd + ["findmnt", dev_path] subprocess.check_call(nsenter_cmd + ["mkfs.ext4", dev_path]) subprocess.check_call(nsenter_cmd + ["mkdir", "-p", mnt_path]) subprocess.check_call(mount_cmd) subprocess.check_call(findmnt_cmd) def checksum_test_file(): read_cmd = nsenter_cmd + ["cat", test_file] data = subprocess.check_output(read_cmd) return checksum_data(str(data).encode('utf-8')) def write_test_file(): # beware don't touch this write command data = random_string(length) write_cmd = [ "/bin/sh -c '/bin/echo", '"' + data + '"', ">", test_file + "'" ] shell_cmd = " ".join(nsenter_cmd + write_cmd) subprocess.check_call(shell_cmd, shell=True) return checksum_test_file() # create snapshot1 with empty fs # NOTE: we cannot use checksum_dev since it assumes # asci data for device data instead of raw bytes snap1 = cmd.snapshot_create(address) # create snapshot2 with a new test file test2_checksum = write_test_file() snap2 = cmd.snapshot_create(address) # create snapshot3 overwriting the test file test3_checksum = write_test_file() snap3 = cmd.snapshot_create(address) # verify existence of the snapshots snapshots = cmd.snapshot_ls(address) assert snap1 in snapshots assert snap2 in snapshots assert snap3 in snapshots # unmount the volume, since each revert will shutdown the device subprocess.check_call(umount_cmd) # restore snapshots 1,2,3 & verify filesystem state print("\nsnapshot_revert_with_frontend snap1 begin") snapshot_revert_with_frontend(address, engine_name, snap1) print("snapshot_revert_with_frontend snap1 finish\n") subprocess.check_call(mount_cmd) # should error since the file does not exist in snapshot 1 with pytest.raises(subprocess.CalledProcessError): print("is expected error, since the file does not exist.") checksum_test_file() subprocess.check_call(umount_cmd) print("\nsnapshot_revert_with_frontend snap2 begin") snapshot_revert_with_frontend(address, engine_name, snap2) print("snapshot_revert_with_frontend snap2 finish\n") subprocess.check_call(mount_cmd) assert checksum_test_file() == test2_checksum subprocess.check_call(umount_cmd) print("\nsnapshot_revert_with_frontend snap3 begin") snapshot_revert_with_frontend(address, engine_name, snap3) print("snapshot_revert_with_frontend snap3 finish\n") subprocess.check_call(mount_cmd) assert checksum_test_file() == test3_checksum subprocess.check_call(umount_cmd) # remove the created mount folder subprocess.check_call(nsenter_cmd + ["rmdir", mnt_path])