def test_scale_up_scale_down(self):
# Test a scale up and scale down. This test creates 2 Chronos nodes,
# and adds 100 timers. It then scales up to 4 Chronos nodes, then
# scales back down to the 2 new Chronos nodes. It then checks that
# 100 timers pop.
# Start initial nodes and add timers
self.write_config_for_nodes([0,1])
start_nodes(0, 2)
create_timers(chronos_nodes[0], 0, 100)
# Scale up
self.write_config_for_nodes([0,1], [2,3])
start_nodes(2, 4)
node_reload_config(0, 2)
node_trigger_scaling(0, 4)
sleep(10)
# Scale down the initial nodes
self.write_config_for_nodes([2,3], [], [0,1])
node_reload_config(0, 4)
node_trigger_scaling(0, 4)
# Check that all the timers have popped
sleep(12)
self.assert_correct_timers_received(100)
def test_gr_no_site_failure(self):
# Test that a GR system works. This test creates 2 Chronos nodes in
# different sites, adds 100 timers, then checks that 100 timers pop.
# Start initial nodes and add timers
self.write_shared_config_for_nodes([0], 'site1', ['site2=127.0.0.12:7254'], 'true')
self.write_shared_config_for_nodes([1], 'site2', ['site1=127.0.0.11:7253'], 'true')
start_nodes(0, 2)
create_timers(chronos_nodes[0], 0, 100)
# Check that all the timers have popped (10 secs with a slight delay
# for replication)
sleep(13)
self.assert_correct_timers_received(100)
def test_deletes(self):
# Test that Chronos replicates deletes from a client. This test creates 3 Chronos
# nodes, adds 100 timers, then deletes the 100 timers again.
# The test checks that we get 0 timer pops
# Start initial nodes and add timers
self.write_config_for_nodes([0,1,2])
start_nodes(0, 3)
create_timers(chronos_nodes[0], 0, 100, "pop", 3)
delete_timers(chronos_nodes[0], 100)
# Check that no timer pops are recieved
sleep(16)
self.assert_correct_timers_received(0)
def test_no_gr(self):
# Test that a system works correctly when GR is disabled. This test
# creates 2 Chronos nodes in different sites, adds 25 timers to each,
# then waits for them to pop, and checks 50 timers have popped.
# Start initial noders and add timers.
# GR is disabled through a shared config option.
self.write_shared_config_for_nodes([0], 'site1', ['site2=127.0.0.12:7254'], 'false')
self.write_shared_config_for_nodes([1], 'site2', ['site1=127.0.0.11:7253'], 'false')
start_nodes(0, 2)
create_timers(chronos_nodes[0], 0, 25)
create_timers(chronos_nodes[1], 25, 50)
# Check that all the timers have popped (after 10 secs with a slight
# delay for replication).
sleep(13)
self.assert_correct_timers_received(50)
def test_gr_deleted_timers(self):
# Test that a GR system works. This test creates 2 Chronos nodes in
# different sites, adds 100 timers from site 1, waits for replication,
# then deletes them from site 2 and checks that no timers pop.
# Start initial nodes and add timers
self.write_shared_config_for_nodes([0], 'site1', ['site2=127.0.0.12:7254'], 'true')
self.write_shared_config_for_nodes([1], 'site2', ['site1=127.0.0.11:7253'], 'true')
start_nodes(0, 2)
create_timers(chronos_nodes[0], 0, 50)
create_timers(chronos_nodes[1], 50, 100)
sleep(3)
# Check that no timers pop (where we'd expect them to take a maximum
# of 10 seconds plus 2 second site delay plus replication delay)
delete_timers(chronos_nodes[1], 100)
sleep(14)
self.assert_correct_timers_received(0)
def test_503_errors(self):
# Test that Chronos replicas pop correctly. This test creates 3 Chronos
# nodes and adds 100 timers that will get 503 errors on their first pop
# and successes on subsequent pops.
# We expect that the primary replica for each timer will get a 503
# error, so the secondary replica will pop and succeed, which should
# prevent the tertiary replica from popping.
# The test checks that we get 100 successful pops and 100 failed pops.
# Start initial nodes and add timers
self.write_config_for_nodes([0, 1, 2])
start_nodes(0, 3)
create_timers(chronos_nodes[0], 0, 100, "pop-with-errors", 3)
# Check that all the timers have popped
sleep(16)
self.assert_correct_timers_received(100)
self.assert_correct_timers_failed(100)
def test_scale_down(self):
# Test that scaling down works. This test creates 4 Chronos nodes,
# adds 100 timers, scales down to 2 Chronos nodes, then checks that
# 100 timers pop.
# Start initial nodes and add timers
self.write_config_for_nodes([0,1,2,3])
start_nodes(0, 4)
create_timers(chronos_nodes[0], 0, 100)
# Scale down
self.write_config_for_nodes([0,1], [], [2,3])
node_reload_config(0, 4)
node_trigger_scaling(0, 4)
kill_nodes(2, 4)
# Check that all the timers have popped
sleep(12)
self.assert_correct_timers_received(100)
def test_scale_up_and_kill(self):
# Test that scaling up definitely moves timers. This test creates 1
# Chronos node and adds 100 timers. It then scales up to 2 Chronos
# nodes, then kills the first node. It then checks all 100 timers pop
# Start initial nodes and add timers
self.write_config_for_nodes([0])
start_nodes(0, 1)
create_timers(chronos_nodes[0], 0, 100)
# Scale up
self.write_config_for_nodes([0], [1])
start_nodes(1, 2)
node_reload_config(0, 1)
node_trigger_scaling(0, 2)
# Now kill the first node
kill_nodes(0, 1)
# Check that all the timers have popped
sleep(12)
self.assert_correct_timers_received(100)
def test_no_gr_site_failures(self):
# Confirm that GR is disabled by checking that the timers on a site are
# lost if that site goes down. This test creates 2 Chronos nodes in
# different sites, adds 25 timers to each site, waits long enough for
# them to be replicated (even though they shouldn't be), kills 1 site,
# and then checks that only 25 timers pop.
# Start initial nodes and add timers.
# GR is disabled through a shared config option.
self.write_shared_config_for_nodes([0], 'site1', ['site2=127.0.0.12:7254','site3=127.0.0.13:7255','site4=127.0.0.14:7256'], 'false')
self.write_shared_config_for_nodes([1], 'site2', ['site1=127.0.0.11:7253','site3=127.0.0.13:7255','site4=127.0.0.14:7256'], 'false')
start_nodes(0, 2)
create_timers(chronos_nodes[0], 0, 25)
create_timers(chronos_nodes[1], 25, 50)
sleep(4)
# Now kill the node in the second site, wait a sufficient time for any
# existing timers to pop (10 seconds, plus 6 seconds delay for the site
# failures, plus a slight delay for replication), and check that only
# the timers set in the first site pop.
kill_nodes(1, 2)
sleep(19)
self.assert_correct_timers_received(25)
def test_gr_site_failures(self):
# Test that timers still pop on site failure. This test creates 4
# Chronos nodes in different sites, adds 25 timers to each site, waits
# for these to be replicated, kills sites 1-3, then checks that
# 100 timers pop.
# Start initial nodes and add timers
self.write_shared_config_for_nodes([0], 'site1', ['site2=127.0.0.12:7254','site3=127.0.0.13:7255','site4=127.0.0.14:7256'], 'true')
self.write_shared_config_for_nodes([1], 'site2', ['site1=127.0.0.11:7253','site3=127.0.0.13:7255','site4=127.0.0.14:7256'], 'true')
self.write_shared_config_for_nodes([2], 'site3', ['site1=127.0.0.11:7253','site2=127.0.0.12:7254','site4=127.0.0.14:7256'], 'true')
self.write_shared_config_for_nodes([3], 'site4', ['site1=127.0.0.11:7253','site2=127.0.0.12:7254','site3=127.0.0.13:7255'], 'true')
start_nodes(0, 4)
create_timers(chronos_nodes[0], 0, 25)
create_timers(chronos_nodes[1], 25, 50)
create_timers(chronos_nodes[2], 50, 75)
create_timers(chronos_nodes[3], 75, 100)
sleep(4)
# Now kill all but a single site, and check that all timers pop within
# expected delay (10 seconds, plus 6 seconds delay for the site
# failures, plus a slight delay for replication)
kill_random_nodes(3)
sleep(19)
self.assert_correct_timers_received(100)