def test_upgrade(dev): # NOQA
offset = 0
length = 128
data = common.random_string(length)
common.verify_data(dev, offset, data)
# both set pointed to the same volume underlying
replicas = [common.REPLICA1, common.REPLICA2]
upgrade_replicas = [common.UPGRADE_REPLICA1, common.UPGRADE_REPLICA2]
launcher.upgrade(common.LONGHORN_UPGRADE_BINARY, upgrade_replicas)
common.verify_data(dev, offset, data)
# cannot start with same binary
with pytest.raises(subprocess.CalledProcessError):
launcher.upgrade(common.LONGHORN_BINARY, replicas)
common.verify_data(dev, offset, data)
# cannot start with wrong replica, would trigger rollback
with pytest.raises(subprocess.CalledProcessError):
launcher.upgrade(common.LONGHORN_UPGRADE_BINARY, "random")
common.verify_data(dev, offset, data)
launcher.upgrade(common.LONGHORN_UPGRADE_BINARY, replicas)
common.verify_data(dev, offset, data)
def test_ha_single_replica_failure(controller, replica1, replica2): # NOQA
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[
common.REPLICA1,
common.REPLICA2
])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
cleanup_replica(replica2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "ERR")
verify_read(dev, data_offset, data)
def test_frontend_switch(
grpc_controller_no_frontend, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
open_replica(grpc_replica1)
open_replica(grpc_replica2)
replicas = grpc_controller_no_frontend.replica_list()
assert len(replicas) == 0
r1_url = grpc_replica1.url
r2_url = grpc_replica2.url
v = grpc_controller_no_frontend.volume_start(replicas=[r1_url, r2_url])
assert v.name == VOLUME_NO_FRONTEND_NAME
assert v.replicaCount == 2
assert v.frontend == ""
start_engine_frontend(ENGINE_NO_FRONTEND_NAME)
dev = get_blockdev(volume=VOLUME_NO_FRONTEND_NAME)
data = random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
shutdown_engine_frontend(ENGINE_NO_FRONTEND_NAME)
start_engine_frontend(ENGINE_NO_FRONTEND_NAME)
dev = get_blockdev(volume=VOLUME_NO_FRONTEND_NAME)
verify_read(dev, data_offset, data)
shutdown_engine_frontend(ENGINE_NO_FRONTEND_NAME)
def test_ha_single_replica_failure(
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
open_replica(grpc_replica1)
open_replica(grpc_replica2)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
r1_url = grpc_replica1.url
r2_url = grpc_replica2.url
v = grpc_controller.volume_start(replicas=[r1_url, r2_url])
assert v.replicaCount == 2
replicas = grpc_controller.replica_list()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev(VOLUME_NAME)
data = random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
cleanup_replica(grpc_replica2)
verify_async(dev, 10, 128, 1)
verify_replica_state(grpc_controller, 1, "ERR")
verify_read(dev, data_offset, data)
def test_ha_single_replica_failure(controller, replica1, replica2): # NOQA
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[common.REPLICA1, common.REPLICA2])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
cleanup_replica(replica2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "ERR")
verify_read(dev, data_offset, data)
def test_basic_rw(dev): # NOQA
for i in range(0, 10):
base = random.randint(1, SIZE - PAGE_SIZE)
offset = (base / PAGE_SIZE) * PAGE_SIZE
length = base - offset
data = common.random_string(length)
common.verify_data(dev, offset, data)
def test_upgrade(dev): # NOQA
offset = 0
length = 128
data = common.random_string(length)
common.verify_data(dev, offset, data)
# both set pointed to the same volume underlying
replicas = [common.REPLICA1, common.REPLICA2]
upgrade_replicas = [common.UPGRADE_REPLICA1, common.UPGRADE_REPLICA2]
launcher.upgrade(common.LONGHORN_UPGRADE_BINARY, upgrade_replicas)
common.verify_data(dev, offset, data)
# cannot start with same binary
with pytest.raises(subprocess.CalledProcessError):
launcher.upgrade(common.LONGHORN_BINARY, replicas)
common.verify_data(dev, offset, data)
# cannot start with wrong replica, would trigger rollback
with pytest.raises(subprocess.CalledProcessError):
launcher.upgrade(common.LONGHORN_UPGRADE_BINARY, "random")
common.verify_data(dev, offset, data)
launcher.upgrade(common.LONGHORN_UPGRADE_BINARY, replicas)
common.verify_data(dev, offset, data)
def test_backup_volume_deletion(
grpc_replica1,
grpc_replica2, # NOQA
grpc_controller,
backup_targets): # NOQA
offset = 0
length = 128
address = grpc_controller.address
for backup_target in backup_targets:
dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
snap_data = random_string(length)
verify_data(dev, offset, snap_data)
snap = cmd.snapshot_create(address)
backup_info = create_backup(address, snap, backup_target)
assert backup_info["VolumeName"] == VOLUME_NAME
assert backup_info["Size"] == BLOCK_SIZE_STR
assert snap in backup_info["SnapshotName"]
cmd.backup_volume_rm(address, VOLUME_NAME, backup_target)
info = cmd.backup_volume_list(address, VOLUME_NAME, backup_target)
assert "cannot find" in info[VOLUME_NAME]["Messages"]["error"]
cmd.sync_agent_server_reset(address)
cleanup_controller(grpc_controller)
cleanup_replica(grpc_replica1)
cleanup_replica(grpc_replica2)
def test_rw_with_metric(
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
rw_dev = get_dev(grpc_replica1, grpc_replica2, grpc_controller)
replies = grpc_controller.metric_get()
# skip the first metric since its fields are 0
next(replies).metric
for i in range(0, 5):
base = random.randint(1, SIZE - PAGE_SIZE)
offset = (base / PAGE_SIZE) * PAGE_SIZE
length = base - offset
data = random_string(length)
verify_data(rw_dev, offset, data)
while 1:
try:
metric = next(replies).metric
# it's hard to confirm the accurate value of metric
assert metric.readBandwidth != 0
assert metric.writeBandwidth != 0
assert metric.iOPS != 0
break
except StopIteration:
time.sleep(1)
def test_ha_single_replica_rebuild(controller, replica1, replica2): # NOQA
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[common.REPLICA1, common.REPLICA2])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
# Cleanup replica2
cleanup_replica(replica2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "ERR")
verify_read(dev, data_offset, data)
controller.delete(replicas[1])
# Rebuild replica2
common.open_replica(replica2)
cmd.add_replica(common.REPLICA2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "RW")
verify_read(dev, data_offset, data)
# WORKAROUND for unable to remove the parent of volume head
newsnap = cmd.snapshot_create()
info = cmd.snapshot_info()
assert len(info) == 3
sysnap = info[newsnap]["parent"]
assert info[sysnap]["parent"] == ""
assert newsnap in info[sysnap]["children"]
assert info[sysnap]["usercreated"] is False
assert info[sysnap]["removed"] is False
cmd.snapshot_purge()
info = cmd.snapshot_info()
assert len(info) == 2
assert info[newsnap] is not None
assert info[VOLUME_HEAD] is not None
def test_basic_rw(dev): # NOQA
for i in range(0, 10):
base = random.randint(0, SIZE - PAGE_SIZE)
offset = (base / PAGE_SIZE) * PAGE_SIZE
length = base - offset
if length == 0:
length = 128
data = common.random_string(length)
common.verify_data(dev, offset, data)
def test_snapshot_tree_rebuild(
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
address = grpc_controller.address
offset = 0
length = 128
open_replica(grpc_replica1)
open_replica(grpc_replica2)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
r1_url = grpc_replica1.url
r2_url = grpc_replica2.url
v = grpc_controller.volume_start(replicas=[r1_url, r2_url])
assert v.name == VOLUME_NAME
assert v.replicaCount == 2
replicas = grpc_controller.replica_list()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev(VOLUME_NAME)
snap, snap_data = snapshot_tree_build(dev, address, ENGINE_NAME, offset,
length)
data = random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
# Cleanup replica2
cleanup_replica(grpc_replica2)
verify_async(dev, 10, 128, 1)
verify_replica_state(grpc_controller, 1, "ERR")
verify_read(dev, data_offset, data)
grpc_controller.replica_delete(replicas[1].address)
# Rebuild replica2
open_replica(grpc_replica2)
cmd.add_replica(address, r2_url)
verify_async(dev, 10, 128, 1)
verify_replica_state(grpc_controller, 1, "RW")
snapshot_tree_verify(dev, address, ENGINE_NAME, offset, length, snap,
snap_data)
def backup_hole_with_backing_file_test(
backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
offset1 = 512
length1 = 256
offset2 = 640
length2 = 256
boundary_offset = 0
boundary_length = 4100 # just pass 4096 into next 4k
hole_offset = 2 * 1024 * 1024
hole_length = 1024
snap1_data = random_string(length1)
verify_data(dev, offset1, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
boundary_data_backup1 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup1 = read_dev(dev, hole_offset, hole_length)
backup1_info = create_backup(address, snap1, backup_target)
snap2_data = random_string(length2)
verify_data(dev, offset2, snap2_data)
snap2_checksum = checksum_dev(dev)
snap2 = cmd.snapshot_create(address)
boundary_data_backup2 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup2 = read_dev(dev, hole_offset, hole_length)
backup2_info = create_backup(address, snap2, backup_target)
restore_with_frontend(address, ENGINE_BACKING_NAME, backup1_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup1
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup1
c = checksum_dev(dev)
assert c == snap1_checksum
restore_with_frontend(address, ENGINE_BACKING_NAME, backup2_info["URL"])
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup2
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup2
c = checksum_dev(dev)
assert c == snap2_checksum
def test_beyond_boundary(dev): # NOQA
# check write at the boundary
data = common.random_string(128)
common.verify_data(dev, SIZE - 128, data)
# out of bounds
with pytest.raises(EnvironmentError) as err:
write_dev(dev, SIZE, "1")
assert 'No space left' in str(err.value)
assert len(read_dev(dev, SIZE, 1)) == 0
# normal writes to verify controller/replica survival
test_basic_rw(dev)
def test_beyond_boundary(dev): # NOQA
# check write at the boundary
data = common.random_string(128)
common.verify_data(dev, SIZE - 128, data)
# out of bounds
with pytest.raises(EnvironmentError) as err:
write_dev(dev, SIZE, "1")
assert 'No space left' in str(err.value)
assert len(read_dev(dev, SIZE, 1)) == 0
# normal writes to verify controller/replica survival
test_basic_rw(dev)
def test_snapshot_tree_rebuild(
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
offset = 0
length = 128
open_replica(grpc_replica1)
open_replica(grpc_replica2)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
v = grpc_controller.volume_start(
replicas=[common.REPLICA1, common.REPLICA2])
assert v.replicaCount == 2
replicas = grpc_controller.replica_list()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
snap, snap_data = snapshot_tree_build(dev, offset, length)
data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
# Cleanup replica2
cleanup_replica(grpc_replica2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(grpc_controller, 1, "ERR")
verify_read(dev, data_offset, data)
grpc_controller.replica_delete(replicas[1].address)
# Rebuild replica2
open_replica(grpc_replica2)
cmd.add_replica(common.REPLICA2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(grpc_controller, 1, "RW")
snapshot_tree_verify(dev, offset, length, snap, snap_data)
def test_snapshot_tree_rebuild(controller, replica1, replica2): # NOQA
offset = 0
length = 128
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[
common.REPLICA1,
common.REPLICA2
])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
snap, snap_data = snapshot_tree_build(dev, offset, length)
data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
# Cleanup replica2
cleanup_replica(replica2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "ERR")
verify_read(dev, data_offset, data)
controller.delete(replicas[1])
# Rebuild replica2
common.open_replica(replica2)
cmd.add_replica(common.REPLICA2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "RW")
snapshot_tree_verify(dev, offset, length, snap, snap_data)
def test_backup_hole_with_backing_file(backing_dev): # NOQA
dev = backing_dev # NOQA
offset1 = 512
length1 = 256
offset2 = 640
length2 = 256
boundary_offset = 0
boundary_length = 4100 # just pass 4096 into next 4k
hole_offset = 2 * 1024 * 1024
hole_length = 1024
snap1_data = common.random_string(length1)
common.verify_data(dev, offset1, snap1_data)
snap1_checksum = common.checksum_dev(dev)
snap1 = cmd.snapshot_create()
boundary_data_backup1 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup1 = read_dev(dev, hole_offset, hole_length)
backup1 = cmd.backup_create(snap1, BACKUP_DEST)
snap2_data = common.random_string(length2)
common.verify_data(dev, offset2, snap2_data)
snap2_checksum = common.checksum_dev(dev)
snap2 = cmd.snapshot_create()
boundary_data_backup2 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup2 = read_dev(dev, hole_offset, hole_length)
backup2 = cmd.backup_create(snap2, BACKUP_DEST)
cmd.backup_restore(backup1)
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup1
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup1
c = common.checksum_dev(dev)
assert c == snap1_checksum
cmd.backup_restore(backup2)
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup2
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup2
c = common.checksum_dev(dev)
assert c == snap2_checksum
def backup_hole_with_backing_file_test(backing_dev, backup_target): # NOQA
dev = backing_dev # NOQA
offset1 = 512
length1 = 256
offset2 = 640
length2 = 256
boundary_offset = 0
boundary_length = 4100 # just pass 4096 into next 4k
hole_offset = 2 * 1024 * 1024
hole_length = 1024
snap1_data = common.random_string(length1)
common.verify_data(dev, offset1, snap1_data)
snap1_checksum = common.checksum_dev(dev)
snap1 = cmd.snapshot_create()
boundary_data_backup1 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup1 = read_dev(dev, hole_offset, hole_length)
backup1 = cmd.backup_create(snap1, backup_target)
snap2_data = common.random_string(length2)
common.verify_data(dev, offset2, snap2_data)
snap2_checksum = common.checksum_dev(dev)
snap2 = cmd.snapshot_create()
boundary_data_backup2 = read_dev(dev, boundary_offset, boundary_length)
hole_data_backup2 = read_dev(dev, hole_offset, hole_length)
backup2 = cmd.backup_create(snap2, backup_target)
cmd.backup_restore(backup1)
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup1
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup1
c = common.checksum_dev(dev)
assert c == snap1_checksum
cmd.backup_restore(backup2)
readed = read_dev(dev, boundary_offset, boundary_length)
assert readed == boundary_data_backup2
readed = read_dev(dev, hole_offset, hole_length)
assert readed == hole_data_backup2
c = common.checksum_dev(dev)
assert c == snap2_checksum
def test_beyond_boundary(dev): # NOQA
# check write at the boundary
data = common.random_string(128)
common.verify_data(dev, SIZE - 128, data)
# out of bounds
with pytest.raises(IOError) as err:
write_dev(dev, SIZE, "1")
assert 'No space left' in str(err.value)
assert len(read_dev(dev, SIZE, 1)) == 0
# normal writes to verify controller/replica survival
for i in range(0, 10):
offset = random.randint(0, SIZE - 256)
length = random.randint(0, 256)
data = common.random_string(length)
common.verify_data(dev, offset, data)
def test_frontend_switch(grpc_controller_no_frontend, # NOQA
grpc_replica1, grpc_replica2): # NOQA
open_replica(grpc_replica1)
open_replica(grpc_replica2)
replicas = grpc_controller_no_frontend.replica_list()
assert len(replicas) == 0
v = grpc_controller_no_frontend.volume_start(replicas=[
common.REPLICA1,
common.REPLICA2
])
assert v.replicaCount == 2
assert v.frontend == ""
launcher.start_engine_frontend(FRONTEND_TGT_BLOCKDEV,
url=LAUNCHER_NO_FRONTEND)
v = grpc_controller_no_frontend.volume_get()
assert v.frontend != ""
dev = get_blockdev(volume=VOLUME2_NAME)
data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
launcher.shutdown_engine_frontend(url=LAUNCHER_NO_FRONTEND)
v = grpc_controller_no_frontend.volume_get()
assert v.frontend != ""
assert v.frontendState == "down"
launcher.start_engine_frontend(FRONTEND_TGT_BLOCKDEV,
url=LAUNCHER_NO_FRONTEND)
v = grpc_controller_no_frontend.volume_get()
assert v.frontend != ""
assert v.frontendState == "up"
dev = get_blockdev(volume=VOLUME2_NAME)
verify_read(dev, data_offset, data)
launcher.shutdown_engine_frontend(url=LAUNCHER_NO_FRONTEND)
def backup_test(
dev,
address, # NOQA
volume_name,
engine_name,
backup_target):
offset = 0
length = 128
snap1_data = random_string(length)
verify_data(dev, offset, snap1_data)
snap1_checksum = checksum_dev(dev)
snap1 = cmd.snapshot_create(address)
backup1_info = create_backup(address, snap1, backup_target)
assert backup1_info["VolumeName"] == volume_name
assert backup1_info["Size"] == BLOCK_SIZE_STR
snap2_data = random_string(length)
verify_data(dev, offset, snap2_data)
snap2_checksum = checksum_dev(dev)
snap2 = cmd.snapshot_create(address)
backup2_info = create_backup(address, snap2, backup_target)
assert backup2_info["VolumeName"] == volume_name
assert backup2_info["Size"] == BLOCK_SIZE_STR
snap3_data = random_string(length)
verify_data(dev, offset, snap3_data)
snap3_checksum = checksum_dev(dev)
snap3 = cmd.snapshot_create(address)
backup3_info = create_backup(address, snap3, backup_target)
assert backup3_info["VolumeName"] == volume_name
assert backup3_info["Size"] == BLOCK_SIZE_STR
restore_with_frontend(address, engine_name, backup3_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap3_data
c = checksum_dev(dev)
assert c == snap3_checksum
rm_backups(address, engine_name, [backup3_info["URL"]])
restore_with_frontend(address, engine_name, backup1_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = checksum_dev(dev)
assert c == snap1_checksum
rm_backups(address, engine_name, [backup1_info["URL"]])
restore_with_frontend(address, engine_name, backup2_info["URL"])
readed = read_dev(dev, offset, length)
assert readed == snap2_data
c = checksum_dev(dev)
assert c == snap2_checksum
rm_backups(address, engine_name, [backup2_info["URL"]])
def test_snapshot_revert(dev): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1 = cmd.snapshot_create()
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create()
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3 = cmd.snapshot_create()
snapList = cmd.snapshot_ls()
assert snap1 in snapList
assert snap2 in snapList
assert snap3 in snapList
cmd.snapshot_revert(snap2)
readed = read_dev(dev, offset, length)
assert readed == snap2_data
cmd.snapshot_revert(snap1)
readed = read_dev(dev, offset, length)
assert readed == snap1_data
def test_snapshot_rm_rolling(dev): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1 = cmd.snapshot_create()
snapList = cmd.snapshot_ls()
assert snap1 in snapList
cmd.snapshot_rm(snap1)
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create()
snapList = cmd.snapshot_ls()
assert snap1 not in snapList
assert snap2 in snapList
# this should trigger real deletion of snap1
cmd.snapshot_rm(snap2)
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3 = cmd.snapshot_create()
snap4_data = common.random_string(length)
common.verify_data(dev, offset, snap4_data)
snap4 = cmd.snapshot_create()
snapList = cmd.snapshot_ls()
assert snap1 not in snapList
assert snap2 not in snapList
assert snap3 in snapList
assert snap4 in snapList
output = cmd.snapshot_info()
info = json.loads(output)
assert info[snap3]["size"] == "4096"
assert info[snap4]["size"] == "4096"
# this should trigger real deletion of snap2 and snap3
cmd.snapshot_rm(snap3)
readed = read_dev(dev, offset, length)
assert readed == snap4_data
def test_ha_remove_extra_disks(
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
address = grpc_controller.address
prepare_backup_dir(BACKUP_DIR)
open_replica(grpc_replica1)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
r1_url = grpc_replica1.url
v = grpc_controller.volume_start(replicas=[r1_url])
assert v.name == VOLUME_NAME
assert v.replicaCount == 1
replicas = grpc_controller.replica_list()
assert len(replicas) == 1
assert replicas[0].mode == "RW"
dev = get_blockdev(VOLUME_NAME)
wasted_data = random_string(128)
data_offset = 1024
verify_data(dev, data_offset, wasted_data)
# now replica1 contains extra data in a snapshot
cmd.snapshot_create(address)
cleanup_controller(grpc_controller)
open_replica(grpc_replica2)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
r2_url = grpc_replica2.url
v = grpc_controller.volume_start(replicas=[r2_url])
assert v.name == VOLUME_NAME
assert v.replicaCount == 1
replicas = grpc_controller.replica_list()
assert len(replicas) == 1
assert replicas[0].mode == "RW"
dev = get_blockdev(VOLUME_NAME)
data = random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
r1 = grpc_replica1.replica_reload()
print(r1)
cmd.add_replica(address, r1_url)
verify_data(dev, data_offset, data)
def test_ha_remove_extra_disks(controller, replica1, replica2): # NOQA
prepare_backup_dir(BACKUP_DIR)
open_replica(replica1)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[
common.REPLICA1,
])
assert v.replicaCount == 1
replicas = controller.list_replica()
assert len(replicas) == 1
assert replicas[0].mode == "RW"
dev = get_blockdev()
wasted_data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, wasted_data)
# now replica1 contains extra data in a snapshot
cmd.snapshot_create()
common.cleanup_controller(controller)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[
common.REPLICA2,
])
assert v.replicaCount == 1
replicas = controller.list_replica()
assert len(replicas) == 1
assert replicas[0].mode == "RW"
dev = get_blockdev()
data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
r = replica1.list_replica()[0]
replica1 = replica1.reload(r)
print(replica1)
cmd.add_replica(common.REPLICA1)
verify_data(dev, data_offset, data)
def test_snapshot_rm_rolling(dev): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1 = cmd.snapshot_create()
snapList = cmd.snapshot_ls()
assert snap1 in snapList
cmd.snapshot_rm(snap1)
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create()
snapList = cmd.snapshot_ls()
assert snap1 not in snapList
assert snap2 in snapList
# this should trigger real deletion of snap1
cmd.snapshot_rm(snap2)
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3 = cmd.snapshot_create()
snap4_data = common.random_string(length)
common.verify_data(dev, offset, snap4_data)
snap4 = cmd.snapshot_create()
snapList = cmd.snapshot_ls()
assert snap1 not in snapList
assert snap2 not in snapList
assert snap3 in snapList
assert snap4 in snapList
# this should trigger real deletion of snap2 and snap3
cmd.snapshot_rm(snap3)
readed = read_dev(dev, offset, length)
assert readed == snap4_data
def test_snapshot_rm_basic(dev): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1 = cmd.snapshot_create()
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create()
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3 = cmd.snapshot_create()
info = cmd.snapshot_info()
assert len(info) == 4
assert VOLUME_HEAD in info
assert snap1 in info
assert snap2 in info
assert snap3 in info
cmd.snapshot_rm(snap2)
cmd.snapshot_purge()
info = cmd.snapshot_info()
assert len(info) == 3
assert snap1 in info
assert snap3 in info
readed = read_dev(dev, offset, length)
assert readed == snap3_data
cmd.snapshot_revert(snap1)
readed = read_dev(dev, offset, length)
assert readed == snap1_data
def restore_to_file_with_backing_file_test(
backup_target, # NOQA
grpc_backing_controller, # NOQA
grpc_backing_replica1, # NOQA
grpc_backing_replica2): # NOQA
address = grpc_backing_controller.address
backing_dev = get_backing_dev(grpc_backing_replica1, grpc_backing_replica2,
grpc_backing_controller)
length0 = 4 * 1024
length1 = 256
length2 = 128
offset0 = 0
offset1 = length1 + offset0
offset2 = length2 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot.
# data in output image == data in backing
check_backing()
check_empty_volume(backing_dev)
snap0 = cmd.snapshot_create(address)
backup = create_backup(address, snap0, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset0, length0)
dev_checksum = checksum_dev(backing_dev)
assert volume_data != ""
assert volume_data == backing_data
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output0_raw = read_file(output_raw_path, offset0, length0)
output0_checksum = checksum_data(read_file(output_raw_path, 0, SIZE))
assert output0_raw == backing_data
assert output0_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output0_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
output0_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output0_qcow2 == backing_data
assert output0_qcow2 == volume_data
assert output0_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
rm_backups(address, ENGINE_BACKING_NAME, [backup])
# create 1 snapshot with 256B data.
# output = snap1(offset0, length1) + backing(offset1, ...)
snap1_data = random_string(length1)
verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
backup = create_backup(address, snap1, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset1, length0 - offset1)
dev_checksum = checksum_dev(backing_dev)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output1_raw_snap1 = read_file(output_raw_path, offset0, length1)
output1_raw_backing = read_file(output_raw_path, offset1,
length0 - offset1)
output1_checksum = checksum_data(read_file(output_raw_path, 0, SIZE))
assert output1_raw_snap1 == snap1_data
assert output1_raw_backing == backing_data
assert output1_raw_snap1 + output1_raw_backing == volume_data
assert output1_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output1_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output1_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output1_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output1_qcow2_snap1 == snap1_data
assert output1_qcow2_backing == backing_data
assert output1_qcow2_snap1 + output1_qcow2_backing == volume_data
assert output1_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
rm_snaps(address, [snap1])
rm_backups(address, ENGINE_BACKING_NAME, [backup])
check_backing()
check_empty_volume(backing_dev)
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2) + backing(offset2, ...)
snap1_data = random_string(length1)
verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create(address)
snap2_data = random_string(length2)
verify_data(backing_dev, offset0, snap2_data)
snap2 = cmd.snapshot_create(address)
backup = create_backup(address, snap2, backup_target)["URL"]
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset1, length0 - offset1)
dev_checksum = checksum_dev(backing_dev)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(output_raw_path, offset0, length2)
output2_raw_snap1 = read_file(output_raw_path, offset2, length1 - length2)
output2_raw_backing = read_file(output_raw_path, offset1,
length0 - offset1)
output2_checksum = checksum_data(read_file(output_raw_path, 0, SIZE))
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset2:length1]
assert output2_raw_backing == backing_data
assert \
volume_data == \
output2_raw_snap2 + output2_raw_snap1 + output2_raw_backing
assert output2_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(address, backup, file(BACKING_FILE_QCOW),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length2)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset2, length1 - length2)
output2_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output2_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output2_qcow2_snap2 == snap2_data
assert output2_qcow2_snap1 == snap1_data[offset2:length1]
assert output2_qcow2_backing == backing_data
assert \
volume_data == \
output2_qcow2_snap2 + output2_qcow2_snap1 + output1_qcow2_backing
assert output2_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
snapshot_revert_with_frontend(address, ENGINE_BACKING_NAME, snap0)
rm_snaps(address, [snap1, snap2])
rm_backups(address, ENGINE_BACKING_NAME, [backup])
check_backing()
check_empty_volume(backing_dev)
def restore_to_file_with_backing_file_test(backing_dev, backup_target):
length0 = 4 * 1024
length1 = 256
length2 = 128
offset0 = 0
offset1 = length1 + offset0
offset2 = length2 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot.
# data in output image == data in backing
check_backing()
check_empty_volume(backing_dev)
snap0 = cmd.snapshot_create()
backup = create_backup(backup_target, snap0)
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(offset0, length0)
dev_checksum = common.checksum_dev(backing_dev)
assert volume_data != ""
assert volume_data == backing_data
cmd.restore_to_file(backup, file(BACKING_FILE_QCOW2),
output_raw_path, IMAGE_FORMAT_RAW)
output0_raw = read_file(output_raw_path, offset0, length0)
output0_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE))
assert output0_raw == backing_data
assert output0_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(backup, file(BACKING_FILE_QCOW2),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output0_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
output0_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output0_qcow2 == backing_data
assert output0_qcow2 == volume_data
assert output0_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
rm_backups([backup])
# create 1 snapshot with 256B data.
# output = snap1(offset0, length1) + backing(offset1, ...)
snap1_data = common.random_string(length1)
common.verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create()
backup = create_backup(backup_target, snap1)
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(
offset1, length0 - offset1)
dev_checksum = common.checksum_dev(backing_dev)
cmd.restore_to_file(backup, file(BACKING_FILE_QCOW2),
output_raw_path, IMAGE_FORMAT_RAW)
output1_raw_snap1 = read_file(
output_raw_path, offset0, length1)
output1_raw_backing = read_file(
output_raw_path, offset1, length0 - offset1)
output1_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE))
assert output1_raw_snap1 == snap1_data
assert output1_raw_backing == backing_data
assert output1_raw_snap1 + output1_raw_backing == volume_data
assert output1_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(backup, file(BACKING_FILE_QCOW2),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output1_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output1_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output1_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output1_qcow2_snap1 == snap1_data
assert output1_qcow2_backing == backing_data
assert output1_qcow2_snap1 + output1_qcow2_backing == volume_data
assert output1_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
cmd.snapshot_revert(snap0)
rm_snaps([snap1])
rm_backups([backup])
check_backing()
check_empty_volume(backing_dev)
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2) + backing(offset2, ...)
snap1_data = common.random_string(length1)
common.verify_data(backing_dev, offset0, snap1_data)
snap1 = cmd.snapshot_create()
snap2_data = common.random_string(length2)
common.verify_data(backing_dev, offset0, snap2_data)
snap2 = cmd.snapshot_create()
backup = create_backup(backup_target, snap2)
volume_data = read_dev(backing_dev, offset0, length0)
backing_data = read_from_backing_file(
offset1, length0 - offset1)
dev_checksum = common.checksum_dev(backing_dev)
cmd.restore_to_file(backup, file(BACKING_FILE_QCOW2),
output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(
output_raw_path, offset0, length2)
output2_raw_snap1 = read_file(
output_raw_path, offset2, length1 - length2)
output2_raw_backing = read_file(
output_raw_path, offset1, length0 - offset1)
output2_checksum = checksum_data(
read_file(output_raw_path, 0, SIZE))
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset2: length1]
assert output2_raw_backing == backing_data
assert \
volume_data == \
output2_raw_snap2 + output2_raw_snap1 + output2_raw_backing
assert output2_checksum == dev_checksum
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(backup, file(BACKING_FILE_QCOW2),
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length2)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset2, length1 - length2)
output2_qcow2_backing = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - offset1)
output2_checksum = checksum_data(
read_qcow2_file_without_backing_file(output_qcow2_path, 0, SIZE))
assert output2_qcow2_snap2 == snap2_data
assert output2_qcow2_snap1 == snap1_data[offset2: length1]
assert output2_qcow2_backing == backing_data
assert \
volume_data == \
output2_qcow2_snap2 + output2_qcow2_snap1 + output1_qcow2_backing
assert output2_checksum == dev_checksum
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
cmd.snapshot_revert(snap0)
rm_snaps([snap1, snap2])
rm_backups([backup])
check_backing()
check_empty_volume(backing_dev)
def snapshot_tree_create_node(dev, offset, length, snap, data, name):
data[name] = common.random_string(length)
common.verify_data(dev, offset, data[name])
snap[name] = cmd.snapshot_create()
def test_snapshot_rm_rolling(dev): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1 = cmd.snapshot_create()
snapList = cmd.snapshot_ls()
assert snap1 in snapList
cmd.snapshot_rm(snap1)
# cannot do anything because it's the parent of volume head
cmd.snapshot_purge()
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create()
info = cmd.snapshot_info()
assert len(info) == 3
assert snap1 in info
assert info[snap1]["removed"] is True
assert snap2 in info
assert info[snap2]["removed"] is False
cmd.snapshot_rm(snap2)
# this should trigger the deletion of snap1
cmd.snapshot_purge()
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3 = cmd.snapshot_create()
snap4_data = common.random_string(length)
common.verify_data(dev, offset, snap4_data)
snap4 = cmd.snapshot_create()
snap5_data = common.random_string(length)
common.verify_data(dev, offset, snap5_data)
snap5 = cmd.snapshot_create()
snapList = cmd.snapshot_ls()
assert snap1 not in snapList
assert snap2 not in snapList
assert snap3 in snapList
assert snap4 in snapList
assert snap5 in snapList
info = cmd.snapshot_info()
assert len(info) == 5
assert snap1 not in info
assert snap2 in info
assert info[snap2]["removed"] is True
assert snap3 in info
assert info[snap3]["size"] == "4096"
assert snap4 in info
assert info[snap4]["size"] == "4096"
assert snap5 in info
assert info[snap5]["size"] == "4096"
cmd.snapshot_rm(snap3)
cmd.snapshot_rm(snap4)
cmd.snapshot_rm(snap5)
# this should trigger the deletion of snap2 - snap4
# and snap5 marked as removed
cmd.snapshot_purge()
info = cmd.snapshot_info()
assert len(info) == 2
assert snap1 not in info
assert snap2 not in info
assert snap3 not in info
assert snap4 not in info
assert snap5 in info
assert info[snap5]["removed"] is True
assert info[snap5]["size"] == "4096"
readed = read_dev(dev, offset, length)
assert readed == snap5_data
def test_ha_double_replica_rebuild(controller, replica1, replica2): # NOQA
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[
common.REPLICA1,
common.REPLICA2
])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
data1 = common.random_string(128)
data1_offset = 1024
verify_data(dev, data1_offset, data1)
# Close replica2
r2 = replica2.list_replica()[0]
assert r2.revisioncounter == 1
r2.close()
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "ERR")
verify_read(dev, data1_offset, data1)
data2 = common.random_string(128)
data2_offset = 512
verify_data(dev, data2_offset, data2)
# Close replica1
r1 = replica1.list_replica()[0]
assert r1.revisioncounter == 12 # 1 + 10 + 1
r1.close()
# Restart volume
common.cleanup_controller(controller)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
# NOTE the order is reversed here
v = v.start(replicas=[
common.REPLICA2,
common.REPLICA1
])
assert v.replicaCount == 2
# replica2 is out because of lower revision counter
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "ERR"
assert replicas[1].mode == "RW"
verify_read(dev, data1_offset, data1)
verify_read(dev, data2_offset, data2)
# Rebuild replica2
r2 = replica2.list_replica()[0]
assert r2.revisioncounter == 1
r2.close()
controller.delete(replicas[0])
cmd.add_replica(common.REPLICA2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "RW")
verify_read(dev, data1_offset, data1)
verify_read(dev, data2_offset, data2)
r1 = replica1.list_replica()[0]
r2 = replica2.list_replica()[0]
assert r1.revisioncounter == 22 # 1 + 10 + 1 + 10
assert r2.revisioncounter == 22 # must be in sync with r1
def test_basic_rw(dev): # NOQA
for i in range(0, 10):
offset = random.randint(0, SIZE - 256)
length = random.randint(0, 256)
data = common.random_string(length)
common.verify_data(dev, offset, data)
def snapshot_tree_create_node(dev, offset, length, snap, data, name):
data[name] = common.random_string(length)
common.verify_data(dev, offset, data[name])
snap[name] = cmd.snapshot_create()
def restore_to_file_without_backing_file_test(dev, backup_target):
length0 = 256
length1 = 128
offset0 = 0
offset1 = length1 + offset0
output_raw_path = file(OUTPUT_FILE_RAW)
output_qcow2_path = file(OUTPUT_FILE_QCOW2)
# create 1 empty snapshot for converting to init state.
snap0 = cmd.snapshot_create()
# create 1 snapshot with 256B data.
# output = snap2(offset0, length1)
snap1_data = common.random_string(length0)
common.verify_data(dev, offset0, snap1_data)
snap1 = cmd.snapshot_create()
backup = create_backup(backup_target, snap1)
cmd.restore_to_file(backup, "",
output_raw_path, IMAGE_FORMAT_RAW)
output1_raw = read_file(output_raw_path, offset0, length0)
assert output1_raw == snap1_data
os.remove(output_raw_path)
assert not os.path.exists(output_raw_path)
cmd.restore_to_file(backup, "",
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output1_qcow2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length0)
assert output1_qcow2 == snap1_data
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
cmd.snapshot_revert(snap0)
rm_snaps([snap1])
rm_backups([backup])
# create 2 snapshots with 256B data and 128B data
# output = snap2(offset0, length1 - length2) +
# snap1(offset2, length2)
snap1_data = common.random_string(length0)
common.verify_data(dev, offset0, snap1_data)
snap1 = cmd.snapshot_create()
snap2_data = common.random_string(length1)
common.verify_data(dev, offset0, snap2_data)
snap2 = cmd.snapshot_create()
backup = create_backup(backup_target, snap2)
cmd.restore_to_file(backup, "",
output_raw_path, IMAGE_FORMAT_RAW)
output2_raw_snap2 = read_file(
output_raw_path, offset0, length1)
output2_raw_snap1 = read_file(
output_raw_path, offset1, length0 - length1)
assert output2_raw_snap2 == snap2_data
assert output2_raw_snap1 == snap1_data[offset1: length0]
cmd.restore_to_file(backup, "",
output_qcow2_path, IMAGE_FORMAT_QCOW2)
output2_qcow2_snap2 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset0, length1)
output2_qcow2_snap1 = read_qcow2_file_without_backing_file(
output_qcow2_path, offset1, length0 - length1)
assert output2_qcow2_snap2 == snap2_data
assert output2_qcow2_snap1 == snap1_data[offset1: length0]
os.remove(output_qcow2_path)
assert not os.path.exists(output_qcow2_path)
cmd.snapshot_revert(snap0)
rm_snaps([snap1, snap2])
rm_backups([backup])
def backup_test(dev, backup_target): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1_checksum = common.checksum_dev(dev)
snap1 = cmd.snapshot_create()
backup1 = cmd.backup_create(snap1, backup_target)
backup1_info = cmd.backup_inspect(backup1)
assert backup1_info["URL"] == backup1
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["VolumeSize"] == VOLUME_SIZE
assert backup1_info["Size"] == BLOCK_SIZE
assert snap1 in backup1_info["SnapshotName"]
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2_checksum = common.checksum_dev(dev)
snap2 = cmd.snapshot_create()
backup2 = cmd.backup_create(snap2, backup_target)
backup2_info = cmd.backup_inspect(backup2)
assert backup2_info["URL"] == backup2
assert backup2_info["VolumeName"] == VOLUME_NAME
assert backup2_info["VolumeSize"] == VOLUME_SIZE
assert backup2_info["Size"] == BLOCK_SIZE
assert snap2 in backup2_info["SnapshotName"]
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3_checksum = common.checksum_dev(dev)
snap3 = cmd.snapshot_create()
backup3 = cmd.backup_create(snap3, backup_target)
backup3_info = cmd.backup_inspect(backup3)
assert backup3_info["URL"] == backup3
assert backup3_info["VolumeName"] == VOLUME_NAME
assert backup3_info["VolumeSize"] == VOLUME_SIZE
assert backup3_info["Size"] == BLOCK_SIZE
assert snap3 in backup3_info["SnapshotName"]
cmd.backup_restore(backup3)
readed = read_dev(dev, offset, length)
assert readed == snap3_data
c = common.checksum_dev(dev)
assert c == snap3_checksum
cmd.backup_rm(backup3)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup3)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(backup3)
cmd.backup_restore(backup1)
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = common.checksum_dev(dev)
assert c == snap1_checksum
cmd.backup_rm(backup1)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup1)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(backup1)
cmd.backup_restore(backup2)
readed = read_dev(dev, offset, length)
assert readed == snap2_data
c = common.checksum_dev(dev)
assert c == snap2_checksum
cmd.backup_rm(backup2)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup2)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_inspect(backup2)
def test_backup(dev): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1 = cmd.snapshot_create()
backup1 = cmd.backup_create(snap1, BACKUP_DEST)
backup1_info = cmd.backup_inspect(backup1)
assert backup1_info["BackupURL"] == backup1
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["VolumeSize"] == VOLUME_SIZE
assert snap1 in backup1_info["SnapshotName"]
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2 = cmd.snapshot_create()
backup2 = cmd.backup_create(snap2, BACKUP_DEST)
backup2_info = cmd.backup_inspect(backup2)
assert backup2_info["BackupURL"] == backup2
assert backup2_info["VolumeName"] == VOLUME_NAME
assert backup2_info["VolumeSize"] == VOLUME_SIZE
assert snap2 in backup2_info["SnapshotName"]
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3 = cmd.snapshot_create()
backup3 = cmd.backup_create(snap3, BACKUP_DEST)
backup3_info = cmd.backup_inspect(backup3)
assert backup3_info["BackupURL"] == backup3
assert backup3_info["VolumeName"] == VOLUME_NAME
assert backup3_info["VolumeSize"] == VOLUME_SIZE
assert snap3 in backup3_info["SnapshotName"]
cmd.backup_restore(backup3)
readed = read_dev(dev, offset, length)
assert readed == snap3_data
cmd.backup_rm(backup3)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup3)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup3)
assert 'cannot find' in str(e.value)
cmd.backup_restore(backup1)
readed = read_dev(dev, offset, length)
assert readed == snap1_data
cmd.backup_rm(backup1)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup1)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup1)
assert 'cannot find' in str(e.value)
cmd.backup_restore(backup2)
readed = read_dev(dev, offset, length)
assert readed == snap2_data
cmd.backup_rm(backup2)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup2)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup2)
assert 'cannot find' in str(e.value)
def test_ha_single_replica_rebuild(controller, replica1, replica2): # NOQA
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[
common.REPLICA1,
common.REPLICA2
])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
# Cleanup replica2
cleanup_replica(replica2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "ERR")
verify_read(dev, data_offset, data)
controller.delete(replicas[1])
# Rebuild replica2
common.open_replica(replica2)
cmd.add_replica(common.REPLICA2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "RW")
verify_read(dev, data_offset, data)
# WORKAROUND for unable to remove the parent of volume head
newsnap = cmd.snapshot_create()
info = cmd.snapshot_info()
assert len(info) == 3
sysnap = info[newsnap]["parent"]
assert info[sysnap]["parent"] == ""
assert newsnap in info[sysnap]["children"]
assert info[sysnap]["usercreated"] is False
assert info[sysnap]["removed"] is False
cmd.snapshot_purge()
info = cmd.snapshot_info()
assert len(info) == 2
assert info[newsnap] is not None
assert info[VOLUME_HEAD] is not None
def test_ha_double_replica_rebuild(
grpc_controller, # NOQA
grpc_replica1,
grpc_replica2): # NOQA
open_replica(grpc_replica1)
open_replica(grpc_replica2)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
v = grpc_controller.volume_start(
replicas=[common.REPLICA1, common.REPLICA2])
assert v.replicaCount == 2
replicas = grpc_controller.replica_list()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
data1 = common.random_string(128)
data1_offset = 1024
verify_data(dev, data1_offset, data1)
# Close replica2
r2 = grpc_replica2.replica_get()
assert r2.revisionCounter == 1
grpc_replica2.replica_close()
verify_async(dev, 10, 128, 1)
common.verify_replica_state(grpc_controller, 1, "ERR")
verify_read(dev, data1_offset, data1)
data2 = common.random_string(128)
data2_offset = 512
verify_data(dev, data2_offset, data2)
# Close replica1
r1 = grpc_replica1.replica_get()
assert r1.revisionCounter == 12 # 1 + 10 + 1
grpc_replica1.replica_close()
# Restart volume
common.cleanup_controller(grpc_controller)
replicas = grpc_controller.replica_list()
assert len(replicas) == 0
# NOTE the order is reversed here
v = grpc_controller.volume_start(
replicas=[common.REPLICA2, common.REPLICA1])
assert v.replicaCount == 2
# replica2 is out because of lower revision counter
replicas = grpc_controller.replica_list()
assert len(replicas) == 2
assert replicas[0].mode == "ERR"
assert replicas[1].mode == "RW"
verify_read(dev, data1_offset, data1)
verify_read(dev, data2_offset, data2)
# Rebuild replica2
r2 = grpc_replica2.replica_get()
assert r2.revisionCounter == 1
grpc_replica2.replica_close()
grpc_controller.replica_delete(replicas[0].address)
cmd.add_replica(common.REPLICA2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(grpc_controller, 1, "RW")
verify_read(dev, data1_offset, data1)
verify_read(dev, data2_offset, data2)
r1 = grpc_replica1.replica_get()
r2 = grpc_replica2.replica_get()
assert r1.revisionCounter == 22 # 1 + 10 + 1 + 10
assert r2.revisionCounter == 22 # must be in sync with r1
def test_backup(dev): # NOQA
offset = 0
length = 128
snap1_data = common.random_string(length)
common.verify_data(dev, offset, snap1_data)
snap1_checksum = common.checksum_dev(dev)
snap1 = cmd.snapshot_create()
backup1 = cmd.backup_create(snap1, BACKUP_DEST)
backup1_info = cmd.backup_inspect(backup1)
assert backup1_info["URL"] == backup1
assert backup1_info["VolumeName"] == VOLUME_NAME
assert backup1_info["VolumeSize"] == VOLUME_SIZE
assert backup1_info["Size"] == BLOCK_SIZE
assert snap1 in backup1_info["SnapshotName"]
snap2_data = common.random_string(length)
common.verify_data(dev, offset, snap2_data)
snap2_checksum = common.checksum_dev(dev)
snap2 = cmd.snapshot_create()
backup2 = cmd.backup_create(snap2, BACKUP_DEST)
backup2_info = cmd.backup_inspect(backup2)
assert backup2_info["URL"] == backup2
assert backup2_info["VolumeName"] == VOLUME_NAME
assert backup2_info["VolumeSize"] == VOLUME_SIZE
assert backup2_info["Size"] == BLOCK_SIZE
assert snap2 in backup2_info["SnapshotName"]
snap3_data = common.random_string(length)
common.verify_data(dev, offset, snap3_data)
snap3_checksum = common.checksum_dev(dev)
snap3 = cmd.snapshot_create()
backup3 = cmd.backup_create(snap3, BACKUP_DEST)
backup3_info = cmd.backup_inspect(backup3)
assert backup3_info["URL"] == backup3
assert backup3_info["VolumeName"] == VOLUME_NAME
assert backup3_info["VolumeSize"] == VOLUME_SIZE
assert backup3_info["Size"] == BLOCK_SIZE
assert snap3 in backup3_info["SnapshotName"]
cmd.backup_restore(backup3)
readed = read_dev(dev, offset, length)
assert readed == snap3_data
c = common.checksum_dev(dev)
assert c == snap3_checksum
cmd.backup_rm(backup3)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup3)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup3)
assert 'cannot find' in str(e.value)
cmd.backup_restore(backup1)
readed = read_dev(dev, offset, length)
assert readed == snap1_data
c = common.checksum_dev(dev)
assert c == snap1_checksum
cmd.backup_rm(backup1)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup1)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup1)
assert 'cannot find' in str(e.value)
cmd.backup_restore(backup2)
readed = read_dev(dev, offset, length)
assert readed == snap2_data
c = common.checksum_dev(dev)
assert c == snap2_checksum
cmd.backup_rm(backup2)
with pytest.raises(subprocess.CalledProcessError):
cmd.backup_restore(backup2)
with pytest.raises(subprocess.CalledProcessError) as e:
cmd.backup_inspect(backup2)
assert 'cannot find' in str(e.value)
def test_ha_double_replica_rebuild(controller, replica1, replica2): # NOQA
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[common.REPLICA1, common.REPLICA2])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
data1 = common.random_string(128)
data1_offset = 1024
verify_data(dev, data1_offset, data1)
# Close replica2
r2 = replica2.list_replica()[0]
assert r2.revisioncounter == '1'
r2.close()
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "ERR")
verify_read(dev, data1_offset, data1)
data2 = common.random_string(128)
data2_offset = 512
verify_data(dev, data2_offset, data2)
# Close replica1
r1 = replica1.list_replica()[0]
assert r1.revisioncounter == '12' # 1 + 10 + 1
r1.close()
# Restart volume
common.cleanup_controller(controller)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
# NOTE the order is reversed here
v = v.start(replicas=[common.REPLICA2, common.REPLICA1])
assert v.replicaCount == 2
# replica2 is out because of lower revision counter
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "ERR"
assert replicas[1].mode == "RW"
verify_read(dev, data1_offset, data1)
verify_read(dev, data2_offset, data2)
# Rebuild replica2
r2 = replica2.list_replica()[0]
assert r2.revisioncounter == '1'
r2.close()
controller.delete(replicas[0])
cmd.add_replica(common.REPLICA2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "RW")
verify_read(dev, data1_offset, data1)
verify_read(dev, data2_offset, data2)
r1 = replica1.list_replica()[0]
r2 = replica2.list_replica()[0]
assert r1.revisioncounter == '22' # 1 + 10 + 1 + 10
assert r2.revisioncounter == '22' # must be in sync with r1
