def test_disconnect(self): msgid = 'msg53' msg = ('goodbye', 'world') command = constants.DISCONNECT queue1_id = 'default' queue2_id = 'scoundrel' workers = ('w1', 'w2', 'w3') schedulers = ('s1', 's2', 's3') self.router.queues = { queue1_id: [(10, workers[0]), (0, workers[1])], queue2_id: [(10, workers[2]), (0, workers[1])] } self.router.workers = { workers[0]: { 'queues': [(10, queue1_id), ], 'hb': monotonic() + 3, 'available_slots': 0, }, workers[1]: { 'queues': [(0, queue1_id), (10, queue2_id)], 'hb': monotonic() + 3, 'available_slots': 2, }, workers[2]: { 'queues': [(10, queue2_id), ], 'hb': monotonic() + 3, 'available_slots': 0, } } self.router.schedulers = { schedulers[0]: { 'hb': 2903.34, }, schedulers[1]: { 'hb': 2902.99, }, schedulers[2]: { 'hb': 2904.00, } } self.router.process_client_message((schedulers[2], '', constants.PROTOCOL_VERSION, command, msgid) + msg) for i in range(0, 3): self.assertNotIn(schedulers[i], self.router.schedulers, "Disconnect failed to remove scheduler {} from " "schedulers.".format(schedulers[i])) self.assertNotIn(workers[i], self.router.workers, "Disconnect failed to remove worker {} from " "workers.".format(workers[i])) self.assertTrue(self.router.received_disconnect, "Router did not " "receive disconnect.")
def test_handle_kbye_from_worker(self): msgid = 'msg10' msg = ('hello', 'world') command = constants.KBYE queue1_id = 'default' queue2_id = 'scallywag' # Worker ids w1 = 'w1' w2 = 'w2' w3 = 'w3' self.router.queues = { queue1_id: [(10, w1), (0, w2)], queue2_id: [(10, w3), (10, w2)], } self.router.workers = { w1: { 'queues': [ (10, queue1_id), ], 'hb': monotonic() + 3, 'available_slots': 0, }, w2: { 'queues': [(0, queue1_id), (10, queue2_id)], 'hb': monotonic() + 3, 'available_slots': 2, }, w3: { 'queues': [ (10, queue2_id), ], 'hb': monotonic() + 3, 'available_slots': 0, } } self.router.process_worker_message((w2, '', constants.PROTOCOL_VERSION, command, msgid) + msg) self.assertNotIn(w2, self.router.queues[queue1_id], "Worker not " "removed from {" "}".format(queue1_id)) self.assertNotIn(w2, self.router.queues[queue2_id], "Worker not " "removed from {" "}".format(queue2_id))
def test_handle_kbye_from_worker(self): msgid = 'msg10' msg = ('hello', 'world') command = constants.KBYE queue1_id = 'default' queue2_id = 'scallywag' # Worker ids w1 = 'w1' w2 = 'w2' w3 = 'w3' self.router.queues = { queue1_id: [(10, w1), (0, w2)], queue2_id: [(10, w3), (10, w2)], } self.router.workers = { w1: { 'queues': [(10, queue1_id), ], 'hb': monotonic() + 3, 'available_slots': 0, }, w2: { 'queues': [(0, queue1_id), (10, queue2_id)], 'hb': monotonic() + 3, 'available_slots': 2, }, w3: { 'queues': [(10, queue2_id), ], 'hb': monotonic() + 3, 'available_slots': 0, } } self.router.process_worker_message((w2, '', constants.PROTOCOL_VERSION, command, msgid) + msg) self.assertNotIn( w2, self.router.queues[queue1_id], "Worker not removed from {}".format(queue1_id)) self.assertNotIn( w2, self.router.queues[queue2_id], "Worker not removed from {}".format(queue2_id))
def test_clean_up_dead_workers(self): worker1_id = 'w1' worker2_id = 'w2' worker3_id = 'w3' queue1_id = 'default' queue2_id = 'jimjam' nonexistent_queue1 = 'nonexistent' t = monotonic() # To ensure the value was changed later because monotonic() is hard to # mock self.assertEqual(self.router._meta['last_worker_cleanup'], 0) conf.HEARTBEAT_TIMEOUT = 1 self.router.queues = { queue1_id: [(10, worker1_id), (0, worker2_id)], queue2_id: [(10, worker3_id), (10, worker2_id)], } self.router.workers = { # 3 in the future worker1_id: { 'queues': [ (10, queue1_id), ], 'hb': t + 3, 'available_slots': 0, }, # below the timeout worker2_id: { 'queues': [(10, queue2_id), (0, queue1_id)], 'hb': t - 2, 'available_slots': 2, }, # below the timeout and a queue missing from self.router.queues worker3_id: { 'queues': [(10, queue2_id), (3, nonexistent_queue1)], 'hb': t - 2, 'available_slots': 0, }, } self.router.clean_up_dead_workers() self.assertIn(worker1_id, self.router.workers) self.assertNotIn(worker2_id, self.router.workers) self.assertNotIn(worker3_id, self.router.workers) self.assertIn(queue1_id, self.router.queues) self.assertNotIn(queue2_id, self.router.queues) self.assertNotIn(nonexistent_queue1, self.router.queues)
def test_clean_up_dead_workers(self): worker1_id = 'w1' worker2_id = 'w2' worker3_id = 'w3' queue1_id = 'default' queue2_id = 'jimjam' nonexistent_queue1 = 'nonexistent' # To ensure the value was changed later because monotonic() is hard to # mock self.assertEqual(self.router._meta['last_worker_cleanup'], 0) conf.HEARTBEAT_TIMEOUT = 1 self.router.queues = { queue1_id: [(10, worker1_id), (0, worker2_id)], queue2_id: [(10, worker3_id), (10, worker2_id)], } self.router.workers = { # 3 in the future worker1_id: { 'queues': [(10, queue1_id), ], 'hb': monotonic() + 3, 'available_slots': 0, }, # below the timeout worker2_id: { 'queues': [(10, queue2_id), (0, queue1_id)], 'hb': 0, 'available_slots': 2, }, # below the timeout and a queue missing from self.router.queues worker3_id: { 'queues': [(10, queue2_id), (3, nonexistent_queue1)], 'hb': 0, 'available_slots': 0, }, } self.router.clean_up_dead_workers() self.assertIn(worker1_id, self.router.workers) self.assertNotIn(worker2_id, self.router.workers) self.assertNotIn(worker3_id, self.router.workers) self.assertIn(queue1_id, self.router.queues) self.assertNotIn(queue2_id, self.router.queues) self.assertNotIn(nonexistent_queue1, self.router.queues)
def test_router_status(self): msgid1 = 'msg21' msgid2 = 'msg19' msgid3 = 'msg6' queue1_id = 'default' queue2_id = 'scallywag' # Worker ids w1 = 'w1' w2 = 'w2' w3 = 'w3' waiting_msg1 = [ '', constants.PROTOCOL_VERSION, 'REQUEST', msgid1, 'kun', '', 'hello world' ] waiting_msg2 = [ '', constants.PROTOCOL_VERSION, 'REQUEST', msgid2, 'kun', '', 'world hello' ] waiting_msg3 = [ '', constants.PROTOCOL_VERSION, 'REQUEST', msgid3, 'blu', '', 'goodbye' ] self.router.queues = { queue1_id: [(10, w1), (0, w2)], queue2_id: [(10, w3), (10, w2)], } t = monotonic() self.router.workers = { w1: { 'queues': [ (10, queue1_id), ], 'hb': t, 'available_slots': 0, }, w2: { 'queues': [(0, queue1_id), (10, queue2_id)], 'hb': t, 'available_slots': 2, }, w3: { 'queues': [ (10, queue2_id), ], 'hb': t, 'available_slots': 0, } } self.router.waiting_messages = { 'kun': EMQdeque(initial=[waiting_msg1, waiting_msg2]), 'blu': EMQdeque(initial=[ waiting_msg3, ]), } # hacky, but the serialize/deserialize converts the keys to unicode # correctly and what not. self.assertEqual( json.loads( json.dumps({ 'job_latencies': self.router.job_latencies, 'executed_functions': self.router.executed_functions, 'waiting_message_counts': [ '{}: {}'.format(q, len(self.router.waiting_messages[q])) for q in self.router.waiting_messages ] # noqa })), json.loads(self.router.get_status())) self.assertEqual( json.loads( json.dumps({ 'connected_workers': self.router.workers, 'connected_queues': self.router.queues })), json.loads(self.router.get_workers_status())) self.assertEqual( json.loads( json.dumps({'connected_schedulers': self.router.schedulers})), json.loads(self.router.get_schedulers_status()))
def test_router_status(self): msgid1 = 'msg21' msgid2 = 'msg19' msgid3 = 'msg6' queue1_id = 'default' queue2_id = 'scallywag' # Worker ids w1 = 'w1' w2 = 'w2' w3 = 'w3' waiting_msg1 = ['', constants.PROTOCOL_VERSION, 'REQUEST', msgid1, 'kun', '', 'hello world'] waiting_msg2 = ['', constants.PROTOCOL_VERSION, 'REQUEST', msgid2, 'kun', '', 'world hello'] waiting_msg3 = ['', constants.PROTOCOL_VERSION, 'REQUEST', msgid3, 'blu', '', 'goodbye'] self.router.queues = { queue1_id: [(10, w1), (0, w2)], queue2_id: [(10, w3), (10, w2)], } t = monotonic() self.router.workers = { w1: { 'queues': [(10, queue1_id), ], 'hb': t, 'available_slots': 0, }, w2: { 'queues': [(0, queue1_id), (10, queue2_id)], 'hb': t, 'available_slots': 2, }, w3: { 'queues': [(10, queue2_id), ], 'hb': t, 'available_slots': 0, } } self.router.waiting_messages = { 'kun': EMQdeque(initial=[waiting_msg1, waiting_msg2]), 'blu': EMQdeque(initial=[waiting_msg3, ]), } # hacky, but the serialize/deserialize converts the keys to unicode # correctly and what not. self.assertEqual( json.loads(json.dumps({ 'job_latencies': self.router.job_latencies, 'executed_functions': self.router.executed_functions, 'waiting_message_counts': [ '{}: {}'.format(q, len(self.router.waiting_messages[q])) for q in self.router.waiting_messages] })), json.loads(self.router.get_status())) self.assertEqual( json.loads(json.dumps({ 'connected_workers': self.router.workers, 'connected_queues': self.router.queues })), json.loads(self.router.get_workers_status())) self.assertEqual( json.loads(json.dumps({ 'connected_schedulers': self.router.schedulers })), json.loads(self.router.get_schedulers_status()))