def test_ha_revision_counter_consistency(controller, replica1,
replica2): # NOQA
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[common.REPLICA1, common.REPLICA2])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
common.verify_async(dev, 10, 128, 100)
r1 = replica1.list_replica()[0]
r2 = replica2.list_replica()[0]
# kernel can merge requests so backend may not receive 1000 writes
assert r1.revisioncounter > '0'
assert r1.revisioncounter == r2.revisioncounter
def test_ha_revision_counter_consistency(controller, replica1, replica2): # NOQA
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[
common.REPLICA1,
common.REPLICA2
])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
common.verify_async(dev, 10, 128, 100)
r1 = replica1.list_replica()[0]
r2 = replica2.list_replica()[0]
# kernel can merge requests so backend may not receive 1000 writes
assert r1.revisioncounter > 0
assert r1.revisioncounter == r2.revisioncounter
def test_ha_remove_extra_disks(controller, replica1, replica2): # NOQA
prepare_backup_dir(BACKUP_DIR)
open_replica(replica1)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[
common.REPLICA1,
])
assert v.replicaCount == 1
replicas = controller.list_replica()
assert len(replicas) == 1
assert replicas[0].mode == "RW"
dev = get_blockdev()
wasted_data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, wasted_data)
# now replica1 contains extra data in a snapshot
cmd.snapshot_create()
common.cleanup_controller(controller)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[
common.REPLICA2,
])
assert v.replicaCount == 1
replicas = controller.list_replica()
assert len(replicas) == 1
assert replicas[0].mode == "RW"
dev = get_blockdev()
data = common.random_string(128)
data_offset = 1024
verify_data(dev, data_offset, data)
r = replica1.list_replica()[0]
replica1 = replica1.reload(r)
print(replica1)
cmd.add_replica(common.REPLICA1)
verify_data(dev, data_offset, data)
def test_ha_double_replica_rebuild(controller, replica1, replica2): # NOQA
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[common.REPLICA1, common.REPLICA2])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
data1 = common.random_string(128)
data1_offset = 1024
verify_data(dev, data1_offset, data1)
# Close replica2
r2 = replica2.list_replica()[0]
assert r2.revisioncounter == '1'
r2.close()
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "ERR")
verify_read(dev, data1_offset, data1)
data2 = common.random_string(128)
data2_offset = 512
verify_data(dev, data2_offset, data2)
# Close replica1
r1 = replica1.list_replica()[0]
assert r1.revisioncounter == '12' # 1 + 10 + 1
r1.close()
# Restart volume
common.cleanup_controller(controller)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
# NOTE the order is reversed here
v = v.start(replicas=[common.REPLICA2, common.REPLICA1])
assert v.replicaCount == 2
# replica2 is out because of lower revision counter
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "ERR"
assert replicas[1].mode == "RW"
verify_read(dev, data1_offset, data1)
verify_read(dev, data2_offset, data2)
# Rebuild replica2
r2 = replica2.list_replica()[0]
assert r2.revisioncounter == '1'
r2.close()
controller.delete(replicas[0])
cmd.add_replica(common.REPLICA2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "RW")
verify_read(dev, data1_offset, data1)
verify_read(dev, data2_offset, data2)
r1 = replica1.list_replica()[0]
r2 = replica2.list_replica()[0]
assert r1.revisioncounter == '22' # 1 + 10 + 1 + 10
assert r2.revisioncounter == '22' # must be in sync with r1
def test_ha_double_replica_rebuild(controller, replica1, replica2): # NOQA
open_replica(replica1)
open_replica(replica2)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
v = v.start(replicas=[
common.REPLICA1,
common.REPLICA2
])
assert v.replicaCount == 2
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "RW"
assert replicas[1].mode == "RW"
dev = get_blockdev()
data1 = common.random_string(128)
data1_offset = 1024
verify_data(dev, data1_offset, data1)
# Close replica2
r2 = replica2.list_replica()[0]
assert r2.revisioncounter == 1
r2.close()
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "ERR")
verify_read(dev, data1_offset, data1)
data2 = common.random_string(128)
data2_offset = 512
verify_data(dev, data2_offset, data2)
# Close replica1
r1 = replica1.list_replica()[0]
assert r1.revisioncounter == 12 # 1 + 10 + 1
r1.close()
# Restart volume
common.cleanup_controller(controller)
replicas = controller.list_replica()
assert len(replicas) == 0
v = controller.list_volume()[0]
# NOTE the order is reversed here
v = v.start(replicas=[
common.REPLICA2,
common.REPLICA1
])
assert v.replicaCount == 2
# replica2 is out because of lower revision counter
replicas = controller.list_replica()
assert len(replicas) == 2
assert replicas[0].mode == "ERR"
assert replicas[1].mode == "RW"
verify_read(dev, data1_offset, data1)
verify_read(dev, data2_offset, data2)
# Rebuild replica2
r2 = replica2.list_replica()[0]
assert r2.revisioncounter == 1
r2.close()
controller.delete(replicas[0])
cmd.add_replica(common.REPLICA2)
verify_async(dev, 10, 128, 1)
common.verify_replica_state(controller, 1, "RW")
verify_read(dev, data1_offset, data1)
verify_read(dev, data2_offset, data2)
r1 = replica1.list_replica()[0]
r2 = replica2.list_replica()[0]
assert r1.revisioncounter == 22 # 1 + 10 + 1 + 10
assert r2.revisioncounter == 22 # must be in sync with r1
