def test_mixed_worker_connections(self):
self.connection_manager.connection_list = []
self.connection_manager.command_handlers = {}
# Spin up a bunch of imaginary gearman connections
good_connection = MockGearmanConnection()
good_connection._should_fail_on_bind = False
good_connection.connected = True
failed_then_retried_connection = MockGearmanConnection()
failed_then_retried_connection._should_fail_on_bind = True
failed_then_retried_connection.connected = True
failed_connection = MockGearmanConnection()
failed_connection._should_fail_on_bind = True
failed_connection.connected = False
# Register all our connections
self.connection_manager.connection_list = [good_connection, failed_then_retried_connection, failed_connection]
# The only alive connections should be the ones that ultimately be connection.connected
alive_connections = self.connection_manager._get_worker_connections()
self.assertTrue(good_connection in alive_connections)
self.assertTrue(failed_then_retried_connection in alive_connections)
self.assertFalse(failed_connection in alive_connections)
def test_establish_worker_connections_dead(self):
self.connection_manager.connection_list = []
self.connection_manager.command_handlers = {}
# We have no connections so there will never be any work to do
self.assertRaises(ServerUnavailable, self.connection_manager.work)
# We were started with a dead connection, make sure we bail again
dead_connection = MockGearmanConnection()
dead_connection._fail_on_bind = True
dead_connection.connected = False
self.connection_manager.connection_list = [dead_connection]
self.assertRaises(ServerUnavailable, self.connection_manager.work)
def test_establish_worker_connections_dead(self):
self.connection_manager.connection_list = []
self.connection_manager.command_handlers = {}
# We have no connections so there will never be any work to do
self.assertRaises(ServerUnavailable, self.connection_manager.work)
# We were started with a dead connection, make sure we bail again
dead_connection = MockGearmanConnection()
dead_connection._fail_on_bind = True
dead_connection.connected = False
self.connection_manager.connection_list = [dead_connection]
self.assertRaises(ServerUnavailable, self.connection_manager.work)
def test_establish_request_connection_dead(self):
self.connection_manager.connection_list = []
self.connection_manager.command_handlers = {}
current_request = self.generate_job_request(submitted=False, accepted=False)
# No connections == death
self.assertRaises(ServerUnavailable, self.connection_manager.establish_request_connection, current_request)
# Spin up a bunch of imaginary gearman connections
failed_connection = MockGearmanConnection()
failed_connection._fail_on_bind = True
self.connection_manager.connection_list.append(failed_connection)
# All failed connections == death
self.assertRaises(ServerUnavailable, self.connection_manager.establish_request_connection, current_request)
def test_worker_already_locked(self):
other_connection = MockGearmanConnection()
self.connection_manager.connection_list.append(other_connection)
self.connection_manager.establish_connection(other_connection)
other_handler = self.connection_manager.connection_to_handler_map[other_connection]
other_handler.recv_command(GEARMAN_COMMAND_NOOP)
# Make sure other handler has a lock
assert self.connection_manager.command_handler_holding_job_lock == other_handler
# Make sure OUR handler has nothing incoming
self.assert_no_pending_commands()
# Make sure we try to grab a job but fail...so go back to sleep
self.command_handler.recv_command(GEARMAN_COMMAND_NOOP)
self.assert_sent_command(GEARMAN_COMMAND_PRE_SLEEP)
# Make sure other handler still has lock
assert self.connection_manager.command_handler_holding_job_lock == other_handler
# Make the other handler release its lock
other_handler.recv_command(GEARMAN_COMMAND_NO_JOB)
# Ensure that the lock has been freed
self.assert_job_lock(is_locked=False)
# Try to do work after we have our lock released
self.move_to_state_wakeup()
self.move_to_state_job_assign_uniq(self.generate_job_dict())
self.move_to_state_wakeup()
self.move_to_state_no_job()
def test_establish_request_connection_dead(self):
self.connection_manager.connection_list = []
self.connection_manager.command_handlers = {}
current_request = self.generate_job_request(submitted=False, accepted=False)
# No connections == death
self.assertRaises(ServerUnavailable, self.connection_manager.establish_request_connection, current_request)
# Spin up a bunch of imaginary gearman connections
failed_connection = MockGearmanConnection()
failed_connection._fail_on_bind = True
self.connection_manager.connection_list.append(failed_connection)
# All failed connections == death
self.assertRaises(ServerUnavailable, self.connection_manager.establish_request_connection, current_request)
def test_no_connections_for_rotation_for_requests(self):
self.connection_manager.connection_list = []
self.connection_manager.command_handlers = {}
current_request = self.generate_job_request()
# No connections == death
self.assertRaises(ServerUnavailable, self.connection_manager._choose_request_connection, current_request)
# Spin up a bunch of imaginary gearman connections
failed_connection = MockGearmanConnection()
failed_connection._should_fail_on_bind = True
failed_connection.connected = False
self.connection_manager.connection_list.append(failed_connection)
# All failed connections == death
self.assertRaises(ServerUnavailable, self.connection_manager._choose_request_connection, current_request)
def test_establish_request_connection_complex(self):
# Spin up a bunch of imaginary gearman connections
failed_connection = MockGearmanConnection()
failed_connection._fail_on_bind = True
failed_then_retried_connection = MockGearmanConnection()
failed_then_retried_connection._fail_on_bind = True
good_connection = MockGearmanConnection()
good_connection.connect()
# Register all our connections
self.connection_manager.connection_list = [
failed_connection, failed_then_retried_connection, good_connection
]
# When we first create our request, our client shouldn't know anything about it
current_request = self.generate_job_request(submitted=False,
accepted=False)
self.failIf(current_request in self.connection_manager.
request_to_rotating_connection_queue)
# Make sure that when we start up, we get our good connection
chosen_connection = self.connection_manager.establish_request_connection(
current_request)
self.assertEqual(chosen_connection, good_connection)
self.assertFalse(failed_connection.connected)
self.assertFalse(failed_then_retried_connection.connected)
self.assertTrue(good_connection.connected)
# No state changed so we should still go to the correct connection
chosen_connection = self.connection_manager.establish_request_connection(
current_request)
self.assertEqual(chosen_connection, good_connection)
# Pretend like our good connection died so we'll need to choose somethign else
good_connection._reset_connection()
good_connection._fail_on_bind = True
failed_then_retried_connection._fail_on_bind = False
failed_then_retried_connection.connect()
# Make sure we rotate good_connection and failed_connection out
chosen_connection = self.connection_manager.establish_request_connection(
current_request)
self.assertEqual(chosen_connection, failed_then_retried_connection)
self.assertFalse(failed_connection.connected)
self.assertTrue(failed_then_retried_connection.connected)
self.assertFalse(good_connection.connected)
def test_establish_worker_connections(self):
self.connection_manager.connection_list = []
self.connection_manager.command_handlers = {}
# Spin up a bunch of imaginary gearman connections
good_connection = MockGearmanConnection()
good_connection.connect()
good_connection._fail_on_bind = False
failed_then_retried_connection = MockGearmanConnection()
failed_then_retried_connection._fail_on_bind = False
failed_connection = MockGearmanConnection()
failed_connection._fail_on_bind = True
# Register all our connections
self.connection_manager.connection_list = [
good_connection, failed_then_retried_connection, failed_connection
]
# The only alive connections should be the ones that ultimately be connection.connected
alive_connections = self.connection_manager.establish_worker_connections(
)
self.assertTrue(good_connection in alive_connections)
self.assertTrue(failed_then_retried_connection in alive_connections)
self.assertFalse(failed_connection in alive_connections)
def test_establish_request_connection_complex(self):
# Spin up a bunch of imaginary gearman connections
failed_connection = MockGearmanConnection()
failed_connection._fail_on_bind = True
failed_then_retried_connection = MockGearmanConnection()
failed_then_retried_connection._fail_on_bind = True
good_connection = MockGearmanConnection()
good_connection.connect()
# Register all our connections
self.connection_manager.connection_list = [failed_connection, failed_then_retried_connection, good_connection]
# When we first create our request, our client shouldn't know anything about it
current_request = self.generate_job_request(submitted=False, accepted=False)
self.failIf(current_request in self.connection_manager.request_to_rotating_connection_queue)
# Make sure that when we start up, we get our good connection
chosen_connection = self.connection_manager.establish_request_connection(current_request)
self.assertEqual(chosen_connection, good_connection)
self.assertFalse(failed_connection.connected)
self.assertFalse(failed_then_retried_connection.connected)
self.assertTrue(good_connection.connected)
# No state changed so we should still go to the correct connection
chosen_connection = self.connection_manager.establish_request_connection(current_request)
self.assertEqual(chosen_connection, good_connection)
# Pretend like our good connection died so we'll need to choose somethign else
good_connection._reset_connection()
good_connection._fail_on_bind = True
failed_then_retried_connection._fail_on_bind = False
failed_then_retried_connection.connect()
# Make sure we rotate good_connection and failed_connection out
chosen_connection = self.connection_manager.establish_request_connection(current_request)
self.assertEqual(chosen_connection, failed_then_retried_connection)
self.assertFalse(failed_connection.connected)
self.assertTrue(failed_then_retried_connection.connected)
self.assertFalse(good_connection.connected)
def test_connection_rotation_for_requests(self):
# Spin up a bunch of imaginary gearman connections
failed_connection = MockGearmanConnection()
failed_connection._should_fail_on_bind = True
failed_connection.connected = False
failed_then_retried_connection = MockGearmanConnection()
failed_then_retried_connection._should_fail_on_connect = True
failed_then_retried_connection.connected = True
good_connection = MockGearmanConnection()
good_connection._should_fail_on_bind = False
failed_then_retried_connection.connected = True
self.connection_manager.connection_list = [failed_connection, failed_then_retried_connection, good_connection]
# Register all our connections
current_request = self.generate_job_request()
self.failIf(current_request in self.connection_manager.request_to_rotating_connection_queue)
# Make sure that when we start up, we get our good connection
chosen_conn = self.connection_manager._choose_request_connection(current_request)
self.assertEqual(chosen_conn, good_connection)
# No state changed so we should still go there
chosen_conn = self.connection_manager._choose_request_connection(current_request)
self.assertEqual(chosen_conn, good_connection)
# Pretend like our good connection died so we'll need to choose somethign else
good_connection._should_fail_on_bind = True
good_connection.connected = False
failed_then_retried_connection._should_fail_on_connect = False
failed_then_retried_connection.connected = True
# Make sure we rotate good_connection and failed_connection out
chosen_conn = self.connection_manager._choose_request_connection(current_request)
self.assertEqual(chosen_conn, failed_then_retried_connection)