def test_disconnect(self):
msgid = 'msg53'
msg = ('goodbye', 'world')
command = constants.DISCONNECT
queue1_id = 'default'
queue2_id = 'scoundrel'
workers = ('w1', 'w2', 'w3')
schedulers = ('s1', 's2', 's3')
self.router.queues = {
queue1_id: [(10, workers[0]), (0, workers[1])],
queue2_id: [(10, workers[2]), (0, workers[1])]
}
self.router.workers = {
workers[0]: {
'queues': [(10, queue1_id), ],
'hb': monotonic() + 3,
'available_slots': 0,
},
workers[1]: {
'queues': [(0, queue1_id), (10, queue2_id)],
'hb': monotonic() + 3,
'available_slots': 2,
},
workers[2]: {
'queues': [(10, queue2_id), ],
'hb': monotonic() + 3,
'available_slots': 0,
}
}
self.router.schedulers = {
schedulers[0]: {
'hb': 2903.34,
},
schedulers[1]: {
'hb': 2902.99,
},
schedulers[2]: {
'hb': 2904.00,
}
}
self.router.process_client_message((schedulers[2], '',
constants.PROTOCOL_VERSION,
command, msgid) + msg)
for i in range(0, 3):
self.assertNotIn(schedulers[i], self.router.schedulers,
"Disconnect failed to remove scheduler {} from "
"schedulers.".format(schedulers[i]))
self.assertNotIn(workers[i], self.router.workers,
"Disconnect failed to remove worker {} from "
"workers.".format(workers[i]))
self.assertTrue(self.router.received_disconnect, "Router did not "
"receive disconnect.")
def test_handle_kbye_from_worker(self):
msgid = 'msg10'
msg = ('hello', 'world')
command = constants.KBYE
queue1_id = 'default'
queue2_id = 'scallywag'
# Worker ids
w1 = 'w1'
w2 = 'w2'
w3 = 'w3'
self.router.queues = {
queue1_id: [(10, w1), (0, w2)],
queue2_id: [(10, w3), (10, w2)],
}
self.router.workers = {
w1: {
'queues': [
(10, queue1_id),
],
'hb': monotonic() + 3,
'available_slots': 0,
},
w2: {
'queues': [(0, queue1_id), (10, queue2_id)],
'hb': monotonic() + 3,
'available_slots': 2,
},
w3: {
'queues': [
(10, queue2_id),
],
'hb': monotonic() + 3,
'available_slots': 0,
}
}
self.router.process_worker_message((w2, '', constants.PROTOCOL_VERSION,
command, msgid) + msg)
self.assertNotIn(w2, self.router.queues[queue1_id], "Worker not "
"removed from {"
"}".format(queue1_id))
self.assertNotIn(w2, self.router.queues[queue2_id], "Worker not "
"removed from {"
"}".format(queue2_id))
def test_handle_kbye_from_worker(self):
msgid = 'msg10'
msg = ('hello', 'world')
command = constants.KBYE
queue1_id = 'default'
queue2_id = 'scallywag'
# Worker ids
w1 = 'w1'
w2 = 'w2'
w3 = 'w3'
self.router.queues = {
queue1_id: [(10, w1), (0, w2)],
queue2_id: [(10, w3), (10, w2)],
}
self.router.workers = {
w1: {
'queues': [(10, queue1_id), ],
'hb': monotonic() + 3,
'available_slots': 0,
},
w2: {
'queues': [(0, queue1_id), (10, queue2_id)],
'hb': monotonic() + 3,
'available_slots': 2,
},
w3: {
'queues': [(10, queue2_id), ],
'hb': monotonic() + 3,
'available_slots': 0,
}
}
self.router.process_worker_message((w2, '', constants.PROTOCOL_VERSION,
command, msgid) + msg)
self.assertNotIn(
w2, self.router.queues[queue1_id],
"Worker not removed from {}".format(queue1_id))
self.assertNotIn(
w2, self.router.queues[queue2_id],
"Worker not removed from {}".format(queue2_id))
def test_clean_up_dead_workers(self):
worker1_id = 'w1'
worker2_id = 'w2'
worker3_id = 'w3'
queue1_id = 'default'
queue2_id = 'jimjam'
nonexistent_queue1 = 'nonexistent'
t = monotonic()
# To ensure the value was changed later because monotonic() is hard to
# mock
self.assertEqual(self.router._meta['last_worker_cleanup'], 0)
conf.HEARTBEAT_TIMEOUT = 1
self.router.queues = {
queue1_id: [(10, worker1_id), (0, worker2_id)],
queue2_id: [(10, worker3_id), (10, worker2_id)],
}
self.router.workers = {
# 3 in the future
worker1_id: {
'queues': [
(10, queue1_id),
],
'hb': t + 3,
'available_slots': 0,
},
# below the timeout
worker2_id: {
'queues': [(10, queue2_id), (0, queue1_id)],
'hb': t - 2,
'available_slots': 2,
},
# below the timeout and a queue missing from self.router.queues
worker3_id: {
'queues': [(10, queue2_id), (3, nonexistent_queue1)],
'hb': t - 2,
'available_slots': 0,
},
}
self.router.clean_up_dead_workers()
self.assertIn(worker1_id, self.router.workers)
self.assertNotIn(worker2_id, self.router.workers)
self.assertNotIn(worker3_id, self.router.workers)
self.assertIn(queue1_id, self.router.queues)
self.assertNotIn(queue2_id, self.router.queues)
self.assertNotIn(nonexistent_queue1, self.router.queues)
def test_clean_up_dead_workers(self):
worker1_id = 'w1'
worker2_id = 'w2'
worker3_id = 'w3'
queue1_id = 'default'
queue2_id = 'jimjam'
nonexistent_queue1 = 'nonexistent'
# To ensure the value was changed later because monotonic() is hard to
# mock
self.assertEqual(self.router._meta['last_worker_cleanup'], 0)
conf.HEARTBEAT_TIMEOUT = 1
self.router.queues = {
queue1_id: [(10, worker1_id), (0, worker2_id)],
queue2_id: [(10, worker3_id), (10, worker2_id)],
}
self.router.workers = {
# 3 in the future
worker1_id: {
'queues': [(10, queue1_id), ],
'hb': monotonic() + 3,
'available_slots': 0,
},
# below the timeout
worker2_id: {
'queues': [(10, queue2_id), (0, queue1_id)],
'hb': 0,
'available_slots': 2,
},
# below the timeout and a queue missing from self.router.queues
worker3_id: {
'queues': [(10, queue2_id), (3, nonexistent_queue1)],
'hb': 0,
'available_slots': 0,
},
}
self.router.clean_up_dead_workers()
self.assertIn(worker1_id, self.router.workers)
self.assertNotIn(worker2_id, self.router.workers)
self.assertNotIn(worker3_id, self.router.workers)
self.assertIn(queue1_id, self.router.queues)
self.assertNotIn(queue2_id, self.router.queues)
self.assertNotIn(nonexistent_queue1, self.router.queues)
def test_router_status(self):
msgid1 = 'msg21'
msgid2 = 'msg19'
msgid3 = 'msg6'
queue1_id = 'default'
queue2_id = 'scallywag'
# Worker ids
w1 = 'w1'
w2 = 'w2'
w3 = 'w3'
waiting_msg1 = [
'', constants.PROTOCOL_VERSION, 'REQUEST', msgid1, 'kun', '',
'hello world'
]
waiting_msg2 = [
'', constants.PROTOCOL_VERSION, 'REQUEST', msgid2, 'kun', '',
'world hello'
]
waiting_msg3 = [
'', constants.PROTOCOL_VERSION, 'REQUEST', msgid3, 'blu', '',
'goodbye'
]
self.router.queues = {
queue1_id: [(10, w1), (0, w2)],
queue2_id: [(10, w3), (10, w2)],
}
t = monotonic()
self.router.workers = {
w1: {
'queues': [
(10, queue1_id),
],
'hb': t,
'available_slots': 0,
},
w2: {
'queues': [(0, queue1_id), (10, queue2_id)],
'hb': t,
'available_slots': 2,
},
w3: {
'queues': [
(10, queue2_id),
],
'hb': t,
'available_slots': 0,
}
}
self.router.waiting_messages = {
'kun': EMQdeque(initial=[waiting_msg1, waiting_msg2]),
'blu': EMQdeque(initial=[
waiting_msg3,
]),
}
# hacky, but the serialize/deserialize converts the keys to unicode
# correctly and what not.
self.assertEqual(
json.loads(
json.dumps({
'job_latencies':
self.router.job_latencies,
'executed_functions':
self.router.executed_functions,
'waiting_message_counts': [
'{}: {}'.format(q,
len(self.router.waiting_messages[q]))
for q in self.router.waiting_messages
] # noqa
})),
json.loads(self.router.get_status()))
self.assertEqual(
json.loads(
json.dumps({
'connected_workers': self.router.workers,
'connected_queues': self.router.queues
})), json.loads(self.router.get_workers_status()))
self.assertEqual(
json.loads(
json.dumps({'connected_schedulers': self.router.schedulers})),
json.loads(self.router.get_schedulers_status()))
def test_router_status(self):
msgid1 = 'msg21'
msgid2 = 'msg19'
msgid3 = 'msg6'
queue1_id = 'default'
queue2_id = 'scallywag'
# Worker ids
w1 = 'w1'
w2 = 'w2'
w3 = 'w3'
waiting_msg1 = ['', constants.PROTOCOL_VERSION, 'REQUEST', msgid1,
'kun', '', 'hello world']
waiting_msg2 = ['', constants.PROTOCOL_VERSION, 'REQUEST', msgid2,
'kun', '', 'world hello']
waiting_msg3 = ['', constants.PROTOCOL_VERSION, 'REQUEST', msgid3,
'blu', '', 'goodbye']
self.router.queues = {
queue1_id: [(10, w1), (0, w2)],
queue2_id: [(10, w3), (10, w2)],
}
t = monotonic()
self.router.workers = {
w1: {
'queues': [(10, queue1_id), ],
'hb': t,
'available_slots': 0,
},
w2: {
'queues': [(0, queue1_id), (10, queue2_id)],
'hb': t,
'available_slots': 2,
},
w3: {
'queues': [(10, queue2_id), ],
'hb': t,
'available_slots': 0,
}
}
self.router.waiting_messages = {
'kun': EMQdeque(initial=[waiting_msg1, waiting_msg2]),
'blu': EMQdeque(initial=[waiting_msg3, ]),
}
# hacky, but the serialize/deserialize converts the keys to unicode
# correctly and what not.
self.assertEqual(
json.loads(json.dumps({
'job_latencies': self.router.job_latencies,
'executed_functions': self.router.executed_functions,
'waiting_message_counts': [
'{}: {}'.format(q,
len(self.router.waiting_messages[q]))
for q in self.router.waiting_messages]
})),
json.loads(self.router.get_status()))
self.assertEqual(
json.loads(json.dumps({
'connected_workers': self.router.workers,
'connected_queues': self.router.queues
})),
json.loads(self.router.get_workers_status()))
self.assertEqual(
json.loads(json.dumps({
'connected_schedulers': self.router.schedulers
})),
json.loads(self.router.get_schedulers_status()))
