def test_snapshot_revert_with_backing_file(grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
offset = 0
length = 256
snap0 = cmd.snapshot_create(address)
before = read_dev(dev, offset, length)
assert before != ""
info = cmd.snapshot_info(address)
assert len(info) == 2
assert VOLUME_HEAD in info
assert snap0 in info
exists = read_from_backing_file(offset, length)
assert before == exists
snapshot_revert_test(dev, address, ENGINE_BACKING_NAME)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
after = read_dev(dev, offset, length)
assert before == after
def rm_snaps(url, snaps):
for s in snaps:
cmd.snapshot_rm(url, s)
cmd.snapshot_purge(url)
wait_for_purge_completion(url)
snap_info_list = cmd.snapshot_info(url)
for s in snaps:
assert s not in snap_info_list
def test_snapshot_rm_basic(grpc_controller, # NOQA
grpc_replica1, grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
existings = {}
snap1 = Snapshot(dev, generate_random_data(existings),
address)
snap2 = Snapshot(dev, generate_random_data(existings),
address)
snap3 = Snapshot(dev, generate_random_data(existings),
address)
info = cmd.snapshot_info(address)
assert len(info) == 4
assert VOLUME_HEAD in info
assert snap1.name in info
assert snap2.name in info
assert snap3.name in info
cmd.snapshot_rm(address, snap2.name)
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
info = cmd.snapshot_info(address)
assert len(info) == 3
assert snap1.name in info
assert snap3.name in info
snap3.verify_checksum()
snap2.verify_data()
snap1.verify_data()
snapshot_revert_with_frontend(address, ENGINE_NAME, snap1.name)
snap3.refute_data()
snap2.refute_data()
snap1.verify_checksum()
def test_ha_single_replica_rebuild(
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
address = grpc_controller.address
open_replica(grpc_replica1)
open_replica(grpc_replica2)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
r1_url = grpc_replica1.url
r2_url = grpc_replica2.url
v = grpc_controller.volume_start(replicas=[r1_url, r2_url])
assert v.replicaCount == 2
replicas = grpc_controller.replica_list()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev(VOLUME_NAME)
data = random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
# Cleanup replica2
cleanup_replica(grpc_replica2)
verify_async(dev, 10, 128, 1)
verify_replica_state(grpc_controller, r2_url, "ERR")
verify_read(dev, data_offset, data)
grpc_controller.replica_delete(replicas[1].address)
# Rebuild replica2
open_replica(grpc_replica2)
cmd.add_replica(address, r2_url)
wait_for_rebuild_complete(address)
verify_async(dev, 10, 128, 1)
verify_replica_state(grpc_controller, r2_url, "RW")
verify_read(dev, data_offset, data)
# WORKAROUND for unable to remove the parent of volume head
newsnap = cmd.snapshot_create(address)
info = cmd.snapshot_info(address)
assert len(info) == 3
sysnap = info[newsnap]["parent"]
assert info[sysnap]["parent"] == ""
assert newsnap in info[sysnap]["children"]
assert info[sysnap]["usercreated"] is False
assert info[sysnap]["removed"] is False
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
info = cmd.snapshot_info(address)
assert len(info) == 2
assert info[newsnap] is not None
assert info[VOLUME_HEAD] is not None
def snapshot_tree_verify_relationship(address, snap, strict):
info = cmd.snapshot_info(address)
assert snap["0a"] in info
assert snap["0b"] in info[snap["0a"]]["children"]
assert snap["0b"] in info
assert info[snap["0b"]]["parent"] == snap["0a"]
assert len(info[snap["0b"]]["children"]) == 3
assert snap["0c"] in info[snap["0b"]]["children"]
assert snap["1a"] in info[snap["0b"]]["children"]
assert snap["2a"] in info[snap["0b"]]["children"]
assert snap["0c"] in info
assert info[snap["0c"]]["parent"] == snap["0b"]
assert not info[snap["0c"]]["children"]
assert snap["1a"] in info
assert info[snap["1a"]]["parent"] == snap["0b"]
assert snap["1b"] in info[snap["1a"]]["children"]
assert snap["1b"] in info
assert info[snap["1b"]]["parent"] == snap["1a"]
assert snap["1c"] in info[snap["1b"]]["children"]
assert snap["1c"] in info
assert info[snap["1c"]]["parent"] == snap["1b"]
assert not info[snap["1c"]]["children"]
assert snap["2a"] in info
assert info[snap["2a"]]["parent"] == snap["0b"]
assert len(info[snap["2a"]]["children"]) == 2
assert snap["2b"] in info[snap["2a"]]["children"]
assert snap["3a"] in info[snap["2a"]]["children"]
assert snap["2b"] in info
assert info[snap["2b"]]["parent"] == snap["2a"]
assert snap["2c"] in info[snap["2b"]]["children"]
assert snap["2c"] in info
assert info[snap["2c"]]["parent"] == snap["2b"]
assert not info[snap["2c"]]["children"]
assert snap["3a"] in info
assert info[snap["3a"]]["parent"] == snap["2a"]
assert snap["3b"] in info[snap["3a"]]["children"]
assert snap["3b"] in info
assert info[snap["3b"]]["parent"] == snap["3a"]
assert snap["3c"] in info[snap["3b"]]["children"]
assert snap["3c"] in info
assert info[snap["3c"]]["parent"] == snap["3b"]
if strict:
assert len(info) == 13
assert info[snap["0a"]]["parent"] == ""
assert VOLUME_HEAD in info[snap["3c"]]["children"]
assert VOLUME_HEAD in info
assert info[VOLUME_HEAD]["parent"] == snap["3c"]
assert not info[VOLUME_HEAD]["children"]
output = cmd.snapshot_ls(address)
assert output == '''ID
{}
{}
{}
{}
{}
{}
'''.format(snap["3c"], snap["3b"], snap["3a"],
snap["2a"], snap["0b"], snap["0a"])
def test_inc_restore_with_rebuild_and_expansion(grpc_controller, grpc_replica1,
grpc_replica2,
grpc_controller_no_frontend,
grpc_fixed_dir_replica1,
grpc_fixed_dir_replica2,
backup_targets): # NOQA
# Pick up a random backup target.
backup_target = backup_targets[random.randint(0, 1)]
address = grpc_controller.address
dr_address = grpc_controller_no_frontend.address
try:
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
except Exception:
pass
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
start_no_frontend_volume(grpc_controller_no_frontend,
grpc_fixed_dir_replica1)
data0_len = random_length(PAGE_SIZE)
data0 = Data(random.randrange(0, SIZE - 2 * PAGE_SIZE, PAGE_SIZE),
data0_len, random_string(data0_len))
snap0 = Snapshot(dev, data0, address)
backup0_info = create_backup(address, snap0.name, backup_target)
assert backup0_info["VolumeName"] == VOLUME_NAME
assert backup0_info["Size"] == str(BLOCK_SIZE)
cmd.backup_restore(dr_address, backup0_info["URL"])
wait_for_restore_completion(dr_address, backup0_info["URL"])
verify_no_frontend_data(data0.offset, data0.content,
grpc_controller_no_frontend)
expand_volume_with_frontend(grpc_controller, EXPANDED_SIZE)
wait_and_check_volume_expansion(grpc_controller, EXPANDED_SIZE)
data1_len = random_length(PAGE_SIZE)
data1 = Data(random.randrange(SIZE, EXPANDED_SIZE - PAGE_SIZE, PAGE_SIZE),
data1_len, random_string(data1_len))
snap1 = Snapshot(dev, data1, address)
backup1_info = create_backup(address, snap1.name, backup_target,
EXPANDED_SIZE_STR)
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["Size"] == str(2 * BLOCK_SIZE)
backup_volumes = cmd.backup_volume_list(address, VOLUME_NAME,
backup_target)
assert VOLUME_NAME in backup_volumes
url = get_backup_volume_url(backup_target, VOLUME_NAME)
backup_info = cmd.backup_inspect_volume(address, url)
assert backup_info["Size"] == EXPANDED_SIZE_STR
# restore command invocation should error out
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup1_info["URL"])
assert "need to expand the DR volume" in e.value.stdout
# The above restore error is triggered before calling the replicas.
# Hence the error won't be recorded in the restore status
# and we can continue restoring backups for the DR volume.
rs = cmd.restore_status(dr_address)
for status in rs.values():
assert status['backupURL'] == backup0_info["URL"]
assert status['lastRestored'] == backup0_info["Name"]
assert 'error' not in status.keys()
assert not status["isRestoring"]
grpc_controller_no_frontend.volume_expand(EXPANDED_SIZE)
wait_for_volume_expansion(grpc_controller_no_frontend, EXPANDED_SIZE)
# This restore command will trigger snapshot purge.
# And the error is triggered before calling the replicas.
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup1_info["URL"])
assert "found more than 1 snapshot in the replicas, " \
"hence started to purge snapshots before the restore" \
in e.value.stdout
wait_for_purge_completion(dr_address)
snaps_info = cmd.snapshot_info(dr_address)
assert len(snaps_info) == 2
volume_head_name = "volume-head"
snap_name = "expand-" + EXPANDED_SIZE_STR
head_info = snaps_info[volume_head_name]
assert head_info["name"] == volume_head_name
assert head_info["parent"] == snap_name
assert not head_info["children"]
assert head_info["usercreated"] is False
snap_info = snaps_info[snap_name]
assert snap_info["name"] == snap_name
assert not snap_info["parent"]
assert volume_head_name in snap_info["children"]
assert snap_info["usercreated"] is False
cmd.backup_restore(dr_address, backup1_info["URL"])
wait_for_restore_completion(dr_address, backup1_info["URL"])
verify_no_frontend_data(data1.offset, data1.content,
grpc_controller_no_frontend)
# For DR volume, the rebuilding replica won't be expanded automatically.
open_replica(grpc_fixed_dir_replica2)
with pytest.raises(subprocess.CalledProcessError):
cmd.add_replica(dr_address, grpc_fixed_dir_replica2.url, True)
# Manually expand the rebuilding replica then retry `add-replica`.
grpc_fixed_dir_replica2.replica_open()
grpc_fixed_dir_replica2.replica_expand(EXPANDED_SIZE)
grpc_fixed_dir_replica2.replica_close()
cmd.add_replica(dr_address, grpc_fixed_dir_replica2.url, True)
replicas = grpc_controller_no_frontend.replica_list()
assert len(replicas) == 2
rw_replica, wo_replica = 0, 0
for r in replicas:
if r.mode == 'RW':
rw_replica += 1
else:
assert r.mode == "WO"
wo_replica += 1
assert rw_replica == 1 and wo_replica == 1
# The old replica will fail the restore but the error won't be recorded.
# Then rebuilding replica will start full restore.
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_restore(dr_address, backup1_info["URL"])
assert "already restored backup" in e.value.stdout
wait_for_restore_completion(dr_address, backup1_info["URL"])
cmd.verify_rebuild_replica(dr_address, grpc_fixed_dir_replica2.url)
replicas = grpc_controller_no_frontend.replica_list()
assert len(replicas) == 2
for r in replicas:
assert r.mode == 'RW'
verify_no_frontend_data(data1.offset, data1.content,
grpc_controller_no_frontend)
cmd.backup_volume_rm(grpc_controller.address, VOLUME_NAME, backup_target)
def test_snapshot_tree_basic(grpc_controller, # NOQA
grpc_replica1, grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
offset = 0
length = 128
snap, data = snapshot_tree_build(dev, address, ENGINE_NAME,
offset, length)
snapshot_revert_with_frontend(address, ENGINE_NAME, snap["1b"])
cmd.snapshot_rm(address, snap["0a"])
cmd.snapshot_rm(address, snap["0b"])
cmd.snapshot_rm(address, snap["1c"])
cmd.snapshot_rm(address, snap["2a"])
cmd.snapshot_rm(address, snap["2b"])
cmd.snapshot_rm(address, snap["2c"])
cmd.snapshot_rm(address, snap["3a"])
cmd.snapshot_rm(address, snap["3b"])
cmd.snapshot_rm(address, snap["3c"])
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
# the result should looks like this
# snap["0b"](r) -> snap["0c"]
# \-> snap["1a"] -> snap["1b"] -> head
info = cmd.snapshot_info(address)
assert len(info) == 5
assert snap["0b"] in info
assert info[snap["0b"]]["parent"] == ""
assert len(info[snap["0b"]]["children"]) == 2
assert snap["0c"] in info[snap["0b"]]["children"]
assert snap["1a"] in info[snap["0b"]]["children"]
assert info[snap["0b"]]["removed"] is True
assert snap["0c"] in info
assert info[snap["0c"]]["parent"] == snap["0b"]
assert not info[snap["0c"]]["children"]
assert snap["1a"] in info
assert info[snap["1a"]]["parent"] == snap["0b"]
assert snap["1b"] in info[snap["1a"]]["children"]
assert snap["1b"] in info
assert info[snap["1b"]]["parent"] == snap["1a"]
assert VOLUME_HEAD in info[snap["1b"]]["children"]
assert VOLUME_HEAD in info
assert info[VOLUME_HEAD]["parent"] == snap["1b"]
snapshot_tree_verify_node(dev, address, ENGINE_NAME,
offset, length, snap, data, "0b")
snapshot_tree_verify_node(dev, address, ENGINE_NAME,
offset, length, snap, data, "0c")
snapshot_tree_verify_node(dev, address, ENGINE_NAME,
offset, length, snap, data, "1a")
snapshot_tree_verify_node(dev, address, ENGINE_NAME,
offset, length, snap, data, "1b")
def test_snapshot_rm_rolling(grpc_controller, # NOQA
grpc_replica1, grpc_replica2): # NOQA
address = grpc_controller.address
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
existings = {}
snap1 = Snapshot(dev, generate_random_data(existings),
address)
snapList = cmd.snapshot_ls(address)
assert snap1.name in snapList
cmd.snapshot_rm(address, snap1.name)
# cannot do anything because it's the parent of volume head
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
snap2 = Snapshot(dev, generate_random_data(existings),
address)
info = cmd.snapshot_info(address)
assert len(info) == 3
assert snap1.name in info
assert snap2.name in info
assert info[snap1.name]["removed"] is True
assert info[snap2.name]["removed"] is False
cmd.snapshot_rm(address, snap2.name)
# this should trigger the deletion of snap1
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
snap2.verify_checksum()
snap1.verify_data()
snap3 = Snapshot(dev, generate_random_data(existings),
address)
snap4 = Snapshot(dev, generate_random_data(existings),
address)
snap5 = Snapshot(dev, generate_random_data(existings),
address)
snapList = cmd.snapshot_ls(address)
assert snap1.name not in snapList
assert snap2.name not in snapList
assert snap3.name in snapList
assert snap4.name in snapList
assert snap5.name in snapList
info = cmd.snapshot_info(address)
assert len(info) == 5
assert snap1.name not in info
assert snap2.name in info
assert snap3.name in info
assert snap4.name in info
assert snap5.name in info
assert info[snap2.name]["removed"] is True
cmd.snapshot_rm(address, snap3.name)
cmd.snapshot_rm(address, snap4.name)
cmd.snapshot_rm(address, snap5.name)
# this should trigger the deletion of snap2 - snap4
# and snap5 marked as removed
cmd.snapshot_purge(address)
wait_for_purge_completion(address)
info = cmd.snapshot_info(address)
assert len(info) == 2
assert snap1.name not in info
assert snap2.name not in info
assert snap3.name not in info
assert snap4.name not in info
assert snap5.name in info
assert info[snap5.name]["removed"] is True
snap5.verify_checksum()
snap4.verify_data()
snap3.verify_data()
snap2.verify_data()
snap1.verify_data()
