def test_snapshot_revert_with_backing_file(grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
offset = 0
length = 256
snap0 = cmd.snapshot_create(address)
before = read_dev(dev, offset, length)
assert before != ""
info = cmd.snapshot_info(address)
assert len(info) == 2
assert VOLUME_HEAD in info
assert snap0 in info
exists = read_from_backing_file(offset, length)
assert before == exists
snapshot_revert_test(dev, address, ENGINE_BACKING_NAME)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
after = read_dev(dev, offset, length)
assert before == after
def snapshot_revert_test(dev, address, engine_name): # NOQA
existings = {}
snap1 = Snapshot(dev, generate_random_data(existings), address)
snap2 = Snapshot(dev, generate_random_data(existings), address)
snap3 = Snapshot(dev, generate_random_data(existings), address)
snapList = cmd.snapshot_ls(address)
assert snap1.name in snapList
assert snap2.name in snapList
assert snap3.name in snapList
snapshot_revert_with_frontend(address, engine_name, snap2.name)
snap3.refute_data()
snap2.verify_checksum()
snap1.verify_data()
snapshot_revert_with_frontend(address, engine_name, snap1.name)
snap3.refute_data()
snap2.refute_data()
snap1.verify_checksum()
def snapshot_tree_build(dev, address, engine_name,
offset, length, strict=True):
# snap["0a"] -> snap["0b"] -> snap["0c"]
# |-> snap["1a"] -> snap["1b"] -> snap["1c"]
# \-> snap["2a"] -> snap["2b"] -> snap["2c"]
# \-> snap["3a"] -> snap["3b"] -> snap["3c"] -> head
snap = {}
data = {}
snapshot_tree_create_node(dev, address, offset, length, snap, data, "0a")
snapshot_tree_create_node(dev, address, offset, length, snap, data, "0b")
snapshot_tree_create_node(dev, address, offset, length, snap, data, "0c")
snapshot_revert_with_frontend(address, engine_name, snap["0b"])
snapshot_tree_create_node(dev, address, offset, length, snap, data, "1a")
snapshot_tree_create_node(dev, address, offset, length, snap, data, "1b")
snapshot_tree_create_node(dev, address, offset, length, snap, data, "1c")
snapshot_revert_with_frontend(address, engine_name, snap["0b"])
snapshot_tree_create_node(dev, address, offset, length, snap, data, "2a")
snapshot_tree_create_node(dev, address, offset, length, snap, data, "2b")
snapshot_tree_create_node(dev, address, offset, length, snap, data, "2c")
snapshot_revert_with_frontend(address, engine_name, snap["2a"])
snapshot_tree_create_node(dev, address, offset, length, snap, data, "3a")
snapshot_tree_create_node(dev, address, offset, length, snap, data, "3b")
snapshot_tree_create_node(dev, address, offset, length, snap, data, "3c")
snapshot_tree_verify(dev, address, engine_name,
offset, length, snap, data, strict)
return snap, data
def test_snapshot_rm_basic(grpc_controller, # NOQA
grpc_replica1, grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
existings = {}
snap1 = Snapshot(dev, generate_random_data(existings),
address)
snap2 = Snapshot(dev, generate_random_data(existings),
address)
snap3 = Snapshot(dev, generate_random_data(existings),
address)
info = cmd.snapshot_info(address)
assert len(info) == 4
assert VOLUME_HEAD in info
assert snap1.name in info
assert snap2.name in info
assert snap3.name in info
cmd.snapshot_rm(address, snap2.name)
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
info = cmd.snapshot_info(address)
assert len(info) == 3
assert snap1.name in info
assert snap3.name in info
snap3.verify_checksum()
snap2.verify_data()
snap1.verify_data()
snapshot_revert_with_frontend(address, ENGINE_NAME, snap1.name)
snap3.refute_data()
snap2.refute_data()
snap1.verify_checksum()
def snapshot_tree_verify_node(dev, address, engine_name,
offset, length, snap, data, name):
snapshot_revert_with_frontend(address, engine_name, snap[name])
readed = read_dev(dev, offset, length)
assert readed == data[name]
def volume_expansion_with_snapshots_test(dev, grpc_controller, # NOQA
volume_name, engine_name,
original_data):
# the default size is 4MB, will expand it to 8MB
address = grpc_controller.address
zero_char = b'\x00'.decode('utf-8')
# write the data to the original part then do expansion
data1_len = random_length(PAGE_SIZE)
data1 = Data(random.randrange(0, SIZE-2*PAGE_SIZE, PAGE_SIZE),
data1_len, random_string(data1_len))
snap1 = Snapshot(dev, data1, address)
expand_volume_with_frontend(grpc_controller, EXPANDED_SIZE)
wait_and_check_volume_expansion(
grpc_controller, EXPANDED_SIZE)
snap1.verify_data()
assert \
dev.readat(0, SIZE) == \
original_data[0:data1.offset] + data1.content + \
original_data[data1.offset+data1.length:]
assert dev.readat(SIZE, SIZE) == zero_char*SIZE
# write the data to both the original part and the expanded part
data2_len = random_length(PAGE_SIZE)
data2 = Data(SIZE-PAGE_SIZE,
data2_len, random_string(data2_len))
snap2 = Snapshot(dev, data2, address)
data3_len = random_length(PAGE_SIZE)
data3 = Data(random.randrange(SIZE, EXPANDED_SIZE-PAGE_SIZE, PAGE_SIZE),
data3_len, random_string(data3_len))
snap3 = Snapshot(dev, data3, address)
snap1.verify_data()
snap2.verify_data()
snap3.verify_data()
assert \
dev.readat(SIZE, SIZE) == zero_char*(data3.offset-SIZE) + \
data3.content + zero_char*(EXPANDED_SIZE-data3.offset-data3.length)
data4_len = random_length(PAGE_SIZE)
data4 = Data(data1.offset,
data4_len, random_string(data4_len))
snap4 = Snapshot(dev, data4, address)
snap4.verify_data()
# revert to snap1 then see if we can still r/w the existing data
# and expanded part
snapshot_revert_with_frontend(address, engine_name, snap1.name)
assert \
dev.readat(0, SIZE) == \
original_data[0:data1.offset] + data1.content + \
original_data[data1.offset+data1.length:]
assert dev.readat(SIZE, SIZE) == zero_char*SIZE
data5_len = random_length(PAGE_SIZE)
data5 = Data(random.randrange(SIZE, EXPANDED_SIZE-PAGE_SIZE, PAGE_SIZE),
data5_len, random_string(data5_len))
snap5 = Snapshot(dev, data5, address)
snap5.verify_data()
assert \
dev.readat(SIZE, SIZE) == zero_char*(data5.offset-SIZE) + \
data5.content + zero_char*(EXPANDED_SIZE-data5.offset-data5.length)
# delete and purge the snap1. it will coalesce with the larger snap2
cmd.snapshot_rm(address, snap1.name)
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
assert \
dev.readat(0, SIZE) == \
original_data[0:data1.offset] + data1.content + \
original_data[data1.offset+data1.length:]
assert \
dev.readat(SIZE, SIZE) == zero_char*(data5.offset-SIZE) + \
data5.content + zero_char*(EXPANDED_SIZE-data5.offset-data5.length)
def test_snapshot_tree_basic(grpc_controller, # NOQA
grpc_replica1, grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
offset = 0
length = 128
snap, data = snapshot_tree_build(dev, address, ENGINE_NAME,
offset, length)
snapshot_revert_with_frontend(address, ENGINE_NAME, snap["1b"])
cmd.snapshot_rm(address, snap["0a"])
cmd.snapshot_rm(address, snap["0b"])
cmd.snapshot_rm(address, snap["1c"])
cmd.snapshot_rm(address, snap["2a"])
cmd.snapshot_rm(address, snap["2b"])
cmd.snapshot_rm(address, snap["2c"])
cmd.snapshot_rm(address, snap["3a"])
cmd.snapshot_rm(address, snap["3b"])
cmd.snapshot_rm(address, snap["3c"])
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
# the result should looks like this
# snap["0b"](r) -> snap["0c"]
# \-> snap["1a"] -> snap["1b"] -> head
info = cmd.snapshot_info(address)
assert len(info) == 5
assert snap["0b"] in info
assert info[snap["0b"]]["parent"] == ""
assert len(info[snap["0b"]]["children"]) == 2
assert snap["0c"] in info[snap["0b"]]["children"]
assert snap["1a"] in info[snap["0b"]]["children"]
assert info[snap["0b"]]["removed"] is True
assert snap["0c"] in info
assert info[snap["0c"]]["parent"] == snap["0b"]
assert not info[snap["0c"]]["children"]
assert snap["1a"] in info
assert info[snap["1a"]]["parent"] == snap["0b"]
assert snap["1b"] in info[snap["1a"]]["children"]
assert snap["1b"] in info
assert info[snap["1b"]]["parent"] == snap["1a"]
assert VOLUME_HEAD in info[snap["1b"]]["children"]
assert VOLUME_HEAD in info
assert info[VOLUME_HEAD]["parent"] == snap["1b"]
snapshot_tree_verify_node(dev, address, ENGINE_NAME,
offset, length, snap, data, "0b")
snapshot_tree_verify_node(dev, address, ENGINE_NAME,
offset, length, snap, data, "0c")
snapshot_tree_verify_node(dev, address, ENGINE_NAME,
offset, length, snap, data, "1a")
snapshot_tree_verify_node(dev, address, ENGINE_NAME,
offset, length, snap, data, "1b")
def restore_to_file_with_backing_file_test(
backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
backing_dev = get_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
length0 = 4 * 1024
length1 = 256
length2 = 128
offset0 = 0
offset1 = length1 + offset0
offset2 = length2 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot.
# data in output image == data in backing
check_backing()
check_empty_volume(backing_dev)
snap0 = cmd.snapshot_create(address)
backup = create_backup(address, snap0, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset0, length0)
dev_checksum = checksum_dev(backing_dev)
assert volume_data != ""
assert volume_data == backing_data
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output0_raw = read_file(output_raw_path, offset0, length0)
output0_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE).encode('utf-8'))
assert output0_raw == backing_data
assert output0_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output0_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
output0_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output0_qcow2.decode('utf-8') == backing_data
assert output0_qcow2.decode('utf-8') == volume_data
assert output0_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
rm_backups(address, ENGINE_BACKING_NAME, [backup])
# create 1 snapshot with 256B data.
# output = snap1(offset0, length1) + backing(offset1, ...)
snap1_data = random_string(length1)
verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
backup = create_backup(address, snap1, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset1, length0 - offset1)
dev_checksum = checksum_dev(backing_dev)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output1_raw_snap1 = read_file(output_raw_path, offset0, length1)
output1_raw_backing = read_file(output_raw_path, offset1,
length0 - offset1)
output1_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE).encode('utf-8'))
assert output1_raw_snap1 == snap1_data
assert output1_raw_backing == backing_data
assert output1_raw_snap1 + output1_raw_backing == volume_data
assert output1_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output1_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output1_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output1_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output1_qcow2_snap1.decode('utf-8') == snap1_data
assert output1_qcow2_backing.decode('utf-8') == backing_data
assert output1_qcow2_snap1.decode('utf-8') + \
output1_qcow2_backing.decode('utf-8') == volume_data
assert output1_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
rm_snaps(address, [snap1])
rm_backups(address, ENGINE_BACKING_NAME, [backup])
check_backing()
check_empty_volume(backing_dev)
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2) + backing(offset2, ...)
snap1_data = random_string(length1)
verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
snap2_data = random_string(length2)
verify_data(backing_dev, offset0, snap2_data)
snap2 = cmd.snapshot_create(address)
backup = create_backup(address, snap2, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset1, length0 - offset1)
dev_checksum = checksum_dev(backing_dev)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(output_raw_path, offset0, length2)
output2_raw_snap1 = read_file(output_raw_path, offset2, length1 - length2)
output2_raw_backing = read_file(output_raw_path, offset1,
length0 - offset1)
output2_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE).encode('utf-8'))
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset2:length1]
assert output2_raw_backing == backing_data
assert \
volume_data == \
output2_raw_snap2 + output2_raw_snap1 + output2_raw_backing
assert output2_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length2)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset2, length1 - length2)
output2_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output2_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output2_qcow2_snap2.decode('utf-8') == snap2_data
assert output2_qcow2_snap1.decode('utf-8') == snap1_data[offset2:length1]
assert output2_qcow2_backing.decode('utf-8') == backing_data
assert \
volume_data == \
output2_qcow2_snap2.decode('utf-8') + \
output2_qcow2_snap1.decode('utf-8') + \
output1_qcow2_backing.decode('utf-8')
assert output2_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
rm_snaps(address, [snap1, snap2])
rm_backups(address, ENGINE_BACKING_NAME, [backup])
check_backing()
check_empty_volume(backing_dev)
def restore_to_file_without_backing_file_test(
backup_target, # NOQA
grpc_controller, # NOQA
grpc_replica1, # NOQA
grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
length0 = 256
length1 = 128
offset0 = 0
offset1 = length1 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot for converting to init state.
snap0 = cmd.snapshot_create(address)
# create 1 snapshot with 256B data.
# output = snap2(offset0, length1)
snap1_data = random_string(length0)
verify_data(dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
backup = create_backup(address, snap1, backup_target)["URL"]
cmd.restore_to_file(address, backup, "", output_raw_path, IMAGE_FORMAT_RAW)
output1_raw = read_file(output_raw_path, offset0, length0)
assert output1_raw == snap1_data
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, "", output_qcow2_path,
IMAGE_FORMAT_QCOW2)
output1_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
assert output1_qcow2.decode('utf-8') == snap1_data
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_NAME, snap0)
rm_snaps(address, [snap1])
rm_backups(address, ENGINE_NAME, [backup])
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2)
snap1_data = random_string(length0)
verify_data(dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
snap2_data = random_string(length1)
verify_data(dev, offset0, snap2_data)
snap2 = cmd.snapshot_create(address)
backup = create_backup(address, snap2, backup_target)["URL"]
cmd.restore_to_file(address, backup, "", output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(output_raw_path, offset0, length1)
output2_raw_snap1 = read_file(output_raw_path, offset1, length0 - length1)
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset1:length0]
cmd.restore_to_file(address, backup, "", output_qcow2_path,
IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - length1)
assert output2_qcow2_snap2.decode('utf-8') == snap2_data
assert output2_qcow2_snap1.decode('utf-8') == snap1_data[offset1:length0]
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_NAME, snap0)
rm_snaps(address, [snap1, snap2])
rm_backups(address, ENGINE_NAME, [backup])
def snapshot_mounted_filesystem_test(volume_name, dev, address,
engine_name): # NOQA
dev_path = dev.dev
mnt_path = "/tmp/mnt-" + volume_name
test_file = mnt_path + "/test"
length = 128
print("dev_path: " + dev_path + "\n")
print("mnt_path: " + mnt_path + "\n")
# create & mount a ext4 filesystem on dev
nsenter_cmd = [
"nsenter", "--mount=/host/proc/1/ns/mnt", "--net=/host/proc/1/ns/net",
"--"
]
mount_cmd = nsenter_cmd + ["mount", "--make-shared", dev_path, mnt_path]
umount_cmd = nsenter_cmd + ["umount", mnt_path]
findmnt_cmd = nsenter_cmd + ["findmnt", dev_path]
subprocess.check_call(nsenter_cmd + ["mkfs.ext4", dev_path])
subprocess.check_call(nsenter_cmd + ["mkdir", "-p", mnt_path])
subprocess.check_call(mount_cmd)
subprocess.check_call(findmnt_cmd)
def checksum_test_file():
read_cmd = nsenter_cmd + ["cat", test_file]
data = subprocess.check_output(read_cmd)
return checksum_data(str(data).encode('utf-8'))
def write_test_file():
# beware don't touch this write command
data = random_string(length)
write_cmd = [
"/bin/sh -c '/bin/echo", '"' + data + '"', ">", test_file + "'"
]
shell_cmd = " ".join(nsenter_cmd + write_cmd)
subprocess.check_call(shell_cmd, shell=True)
return checksum_test_file()
# create snapshot1 with empty fs
# NOTE: we cannot use checksum_dev since it assumes
# asci data for device data instead of raw bytes
snap1 = cmd.snapshot_create(address)
# create snapshot2 with a new test file
test2_checksum = write_test_file()
snap2 = cmd.snapshot_create(address)
# create snapshot3 overwriting the test file
test3_checksum = write_test_file()
snap3 = cmd.snapshot_create(address)
# verify existence of the snapshots
snapshots = cmd.snapshot_ls(address)
assert snap1 in snapshots
assert snap2 in snapshots
assert snap3 in snapshots
# unmount the volume, since each revert will shutdown the device
subprocess.check_call(umount_cmd)
# restore snapshots 1,2,3 & verify filesystem state
print("\nsnapshot_revert_with_frontend snap1 begin")
snapshot_revert_with_frontend(address, engine_name, snap1)
print("snapshot_revert_with_frontend snap1 finish\n")
subprocess.check_call(mount_cmd)
# should error since the file does not exist in snapshot 1
with pytest.raises(subprocess.CalledProcessError):
print("is expected error, since the file does not exist.")
checksum_test_file()
subprocess.check_call(umount_cmd)
print("\nsnapshot_revert_with_frontend snap2 begin")
snapshot_revert_with_frontend(address, engine_name, snap2)
print("snapshot_revert_with_frontend snap2 finish\n")
subprocess.check_call(mount_cmd)
assert checksum_test_file() == test2_checksum
subprocess.check_call(umount_cmd)
print("\nsnapshot_revert_with_frontend snap3 begin")
snapshot_revert_with_frontend(address, engine_name, snap3)
print("snapshot_revert_with_frontend snap3 finish\n")
subprocess.check_call(mount_cmd)
assert checksum_test_file() == test3_checksum
subprocess.check_call(umount_cmd)
# remove the created mount folder
subprocess.check_call(nsenter_cmd + ["rmdir", mnt_path])
