def simple_recovery(self, kill_command):
value = x.rc_client.read(x.table, 'testKey')
expect(value).equals(('testValue', 1))
# find the host corresponding to the server with our table and 'testKey',
# then kill its rc-server!
locator = x.rc_client.testing_get_service_locator(x.table, 'testKey')
host = ctu.get_host(locator)
x.node_containers[host].exec_run(kill_command)
# read the value again (without waiting for the server to recover). It
# should come out to the same value
value = x.rc_client.read(x.table, 'testKey')
expect(value).equals(('testValue', 1))
def test_down_can_still_write(self):
value = x.rc_client.read(x.table, 'testKey')
expect(value).equals(('testValue', 1))
# find the host corresponding to the server with our table and 'testKey',
# then kill its rc-server!
locator = x.rc_client.testing_get_service_locator(x.table, 'testKey')
host = ctu.get_host(locator)
x.node_containers[host].exec_run('killall -SIGKILL rc-server')
# after the master server is down, we try to write (not read). We expect
# the read that follows to correctly contain our value.
x.rc_client.write(x.table, 'testKey', 'testValue2')
value = x.rc_client.read(x.table, 'testKey')
expect(value).equals(('testValue2', 2))
def test_down_can_still_read(self):
value = x.rc_client.read(x.table, 'testKey')
expect(value).equals(('testValue', 1))
# find the host corresponding to the elected coordinator, kill its rc-coordinator!
# we should still be able to get the testKey.
zk_client = ctu.get_zookeeper_client(x.ensemble)
locator = zk_client.get('/ramcloud/main/coordinator')[0]
host = ctu.get_host(locator)
x.node_containers[host].exec_run('killall -SIGKILL rc-coordinator')
# after the coordinator is down, we try to read. We expect
# to see our value.
value = x.rc_client.read(x.table, 'testKey')
time.sleep(
3
) # 3 seconds is needed for a new coordinator to be elected & results to show in zk
new_locator = zk_client.get('/ramcloud/main/coordinator')[0]
expect(value).equals(('testValue', 1))
expect(new_locator).not_equals(None)
expect(new_locator).not_equals(locator)