def test_heartbeat_stopped_and_restored(self):
member2 = self.rc.startMember(self.cluster.id)
addr = Address(member2.host, member2.port)
wait_for_partition_table(self.client)
open_connection_to_address(self.client, member2.uuid)
def connection_collector():
connections = []
def collector(c, *args):
connections.append(c)
collector.connections = connections
return collector
connection_added_collector = connection_collector()
connection_removed_collector = connection_collector()
self.client._connection_manager.add_listener(
connection_added_collector, connection_removed_collector)
self.simulate_heartbeat_lost(self.client, addr, 2)
def assert_heartbeat_stopped_and_restored():
self.assertEqual(1, len(connection_added_collector.connections))
self.assertEqual(1, len(connection_removed_collector.connections))
stopped_connection = connection_added_collector.connections[0]
restored_connection = connection_removed_collector.connections[0]
self.assertEqual(stopped_connection.connected_address,
Address(member2.host, member2.port))
self.assertEqual(restored_connection.connected_address,
Address(member2.host, member2.port))
self.assertTrueEventually(assert_heartbeat_stopped_and_restored)
def test_non_smart_listener_add_member(self):
self.client_config["smart_routing"] = False
client = self.create_client(self.client_config)
map = client.get_map(random_string()).blocking()
map.add_entry_listener(added_func=self.collector)
m3 = self.cluster.start_member()
wait_for_partition_table(client)
key_m3 = generate_key_owned_by_instance(client, m3.uuid)
map.put(key_m3, 'value')
def assert_event():
self.assertEqual(1, len(self.collector.events))
self.assertTrueEventually(assert_event)
def test_smart_listener_remove_member(self):
self.client_config["smart_routing"] = True
client = self.create_client(self.client_config)
wait_for_partition_table(client)
key_m1 = generate_key_owned_by_instance(client, self.m1.uuid)
map = client.get_map(random_string()).blocking()
map.put(key_m1, 'value1')
map.add_entry_listener(updated_func=self.collector)
self.m1.shutdown()
map.put(key_m1, 'value2')
def assert_event():
self.assertEqual(1, len(self.collector.events))
self.assertTrueEventually(assert_event)
def test_non_smart_listener_remove_member(self):
self.client_config.network.smart_routing = False
client = self.create_client(self.client_config)
map = client.get_map(random_string()).blocking()
map.add_entry_listener(added_func=self.collector)
self.m2.shutdown()
wait_for_partition_table(client)
generated_key = generate_key_owned_by_instance(client, self.m1.uuid)
map.put(generated_key, 'value')
def assert_event():
self.assertEqual(1, len(self.collector.events))
self.assertTrueEventually(assert_event)
def test_add_member(self, _, is_smart):
self.client_config["smart_routing"] = is_smart
client = self.create_client(self.client_config)
random_map = client.get_map(random_string()).blocking()
random_map.add_entry_listener(added_func=self.collector,
updated_func=self.collector)
m2 = self.cluster.start_member()
wait_for_partition_table(client)
key_m2 = generate_key_owned_by_instance(client, m2.uuid)
assertion_succeeded = False
def run():
nonlocal assertion_succeeded
# When a new connection is added, we add the existing
# listeners to it, but we do it non-blocking. So, it might
# be the case that, the listener registration request is
# sent to the new member, but not completed yet.
# So, we might not get an event for the put. To avoid this,
# we will put multiple times.
for i in range(10):
if assertion_succeeded:
# We have successfully got an event
return
random_map.put(key_m2, f"value-{i}")
time.sleep((i + 1) * 0.1)
put_thread = threading.Thread(target=run)
put_thread.start()
def assert_event():
nonlocal assertion_succeeded
self.assertGreaterEqual(len(self.collector.events), 1)
assertion_succeeded = True
self.assertTrueEventually(assert_event)
put_thread.join()
def test_heartbeat_stopped_and_restored(self):
member2 = self.rc.startMember(self.cluster.id)
addr = Address(member2.host, member2.port)
wait_for_partition_table(self.client)
open_connection_to_address(self.client, member2.uuid)
def connection_collector():
connections = []
def collector(c, *_):
connections.append(c)
collector.connections = connections
return collector
connection_added_collector = connection_collector()
connection_removed_collector = connection_collector()
self.client._connection_manager.add_listener(
connection_added_collector, connection_removed_collector)
assertion_succeeded = False
def run():
nonlocal assertion_succeeded
# It is possible for client to override the set last_read_time
# of the connection, in case of the periodically sent heartbeat
# requests getting responses, right after we try to set a new
# value to it, before the next iteration of the heartbeat manager.
# In this case, the connection won't be closed, and the test would
# fail. To avoid it, we will try multiple times.
for i in range(10):
if assertion_succeeded:
# We have successfully simulated heartbeat loss
return
for connection in self.client._connection_manager.active_connections.values(
):
if connection.remote_address == addr:
connection.last_read_time -= 2
break
time.sleep((i + 1) * 0.1)
simulation_thread = threading.Thread(target=run)
simulation_thread.start()
def assert_heartbeat_stopped_and_restored():
nonlocal assertion_succeeded
self.assertGreaterEqual(
len(connection_added_collector.connections), 1)
self.assertGreaterEqual(
len(connection_removed_collector.connections), 1)
stopped_connection = connection_removed_collector.connections[0]
restored_connection = connection_added_collector.connections[0]
self.assertEqual(
stopped_connection.connected_address,
Address(member2.host, member2.port),
)
self.assertEqual(
restored_connection.connected_address,
Address(member2.host, member2.port),
)
assertion_succeeded = True
self.assertTrueEventually(assert_heartbeat_stopped_and_restored)
simulation_thread.join()