def test_on_elect_ack_win_but_no_action(self):
c = self.Consumer(hostname='*****@*****.**') # I will win
g = Gossip(c)
g.election_handlers = {}
with patch('celery.worker.consumer.error') as error:
self.setup_election(g, c)
self.assertTrue(error.called)
def test_on_message__task(self):
c = self.Consumer()
g = Gossip(c)
self.assertTrue(g.enabled)
message = Mock(name='message')
message.delivery_info = {'routing_key': 'task.failed'}
g.on_message(Mock(name='prepare'), message)
def test_on_message__task(self):
c = self.Consumer()
g = Gossip(c)
self.assertTrue(g.enabled)
message = Mock(name='message')
message.delivery_info = {'routing_key': 'task.failed'}
g.on_message(Mock(name='prepare'), message)
def test_on_node_lost(self):
c = self.Consumer()
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
with patch('celery.worker.consumer.info') as info:
g.on_node_lost(c)
info.assert_called_with('missed heartbeat from %s', '*****@*****.**')
def test_on_node_leave(self):
c = self.Consumer()
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
with patch("celery.worker.consumer.debug") as debug:
g.on_node_leave(c)
debug.assert_called_with("%s left", "*****@*****.**")
def test_on_node_lost(self):
c = self.Consumer()
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
with patch("celery.worker.consumer.info") as info:
g.on_node_lost(c)
info.assert_called_with("missed heartbeat from %s", "*****@*****.**")
def test_on_elect_ack_win_but_no_action(self):
c = self.Consumer(hostname='*****@*****.**') # I will win
g = Gossip(c)
g.election_handlers = {}
with patch('celery.worker.consumer.error') as error:
self.setup_election(g, c)
self.assertTrue(error.called)
def test_election(self):
c = self.Consumer()
g = Gossip(c)
g.start(c)
g.election("id", "topic", "action")
self.assertListEqual(g.consensus_replies["id"], [])
g.dispatcher.send.assert_called_with("worker-elect", id="id", topic="topic", cver=1, action="action")
def test_on_node_leave(self):
c = self.Consumer()
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
with patch('celery.worker.consumer.debug') as debug:
g.on_node_leave(c)
debug.assert_called_with('%s left', '*****@*****.**')
def test_on_node_join(self):
c = self.Consumer()
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
with patch('celery.worker.consumer.debug') as debug:
g.on_node_join(c)
debug.assert_called_with('%s joined the party', '*****@*****.**')
def test_on_message__task(self):
c = self.Consumer()
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
self.assertTrue(g.enabled)
message = Mock(name="message")
message.delivery_info = {"routing_key": "task.failed"}
g.on_message(Mock(name="prepare"), message)
def test_on_elect_ack_win_but_no_action(self):
c = self.Consumer(hostname="*****@*****.**") # I will win
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
g.election_handlers = {}
with patch("celery.worker.consumer.error") as error:
self.setup_election(g, c)
self.assertTrue(error.called)
def test_register_timer(self):
c = self.Consumer()
g = Gossip(c)
g.register_timer()
c.timer.apply_interval.assert_called_with(g.interval * 1000.0, g.periodic)
tref = g._tref
g.register_timer()
tref.cancel.assert_called_with()
def test_register_timer(self):
c = self.Consumer()
g = Gossip(c)
g.register_timer()
c.timer.call_repeatedly.assert_called_with(g.interval, g.periodic)
tref = g._tref
g.register_timer()
tref.cancel.assert_called_with()
def test_election(self):
c = self.Consumer()
g = Gossip(c)
g.start(c)
g.election('id', 'topic', 'action')
self.assertListEqual(g.consensus_replies['id'], [])
g.dispatcher.send.assert_called_with(
'worker-elect', id='id', topic='topic', cver=1, action='action',
)
def test_on_elect_ack_win(self):
c = self.Consumer(hostname='*****@*****.**') # I will win
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
handler = g.election_handlers['topic'] = Mock()
self.setup_election(g, c)
handler.assert_called_with('action')
def test_on_elect_ack_lose(self):
c = self.Consumer(hostname='*****@*****.**') # I will lose
c.app.connection = _amqp_connection()
g = Gossip(c)
handler = g.election_handlers['topic'] = Mock()
self.setup_election(g, c)
self.assertFalse(handler.called)
def test_register_timer(self):
c = self.Consumer()
g = Gossip(c)
g.register_timer()
c.timer.call_repeatedly.assert_called_with(g.interval, g.periodic)
tref = g._tref
g.register_timer()
tref.cancel.assert_called_with()
def test_on_elect(self):
c = self.Consumer()
g = Gossip(c)
g.start(c)
event = self.Event('id1')
g.on_elect(event)
in_heap = g.consensus_requests['id1']
self.assertTrue(in_heap)
g.dispatcher.send.assert_called_with('worker-elect-ack', id='id1')
event.pop('clock')
with patch('celery.worker.consumer.error') as error:
g.on_elect(event)
self.assertTrue(error.called)
def test_call_task(self):
c = self.Consumer()
g = Gossip(c)
g.start(c)
with patch('celery.worker.consumer.signature') as signature:
sig = signature.return_value = Mock()
task = Mock()
g.call_task(task)
signature.assert_called_with(task, app=c.app)
sig.apply_async.assert_called_with()
sig.apply_async.side_effect = MemoryError()
with patch('celery.worker.consumer.error') as error:
g.call_task(task)
self.assertTrue(error.called)
def test_election(self):
c = self.Consumer()
c.app.connection = _amqp_connection()
g = Gossip(c)
g.start(c)
g.election('id', 'topic', 'action')
self.assertListEqual(g.consensus_replies['id'], [])
g.dispatcher.send.assert_called_with(
'worker-elect', id='id', topic='topic', cver=1, action='action',
)
def test_register_timer(self):
c = self.Consumer()
g = Gossip(c)
g.register_timer()
c.timer.apply_interval.assert_called_with(
g.interval * 1000.0,
g.periodic,
)
tref = g._tref
g.register_timer()
tref.cancel.assert_called_with()
def test_on_elect(self):
c = self.Consumer()
g = Gossip(c)
g.start(c)
event = self.Event("id1")
g.on_elect(event)
in_heap = g.consensus_requests["id1"]
self.assertTrue(in_heap)
g.dispatcher.send.assert_called_with("worker-elect-ack", id="id1")
event.pop("clock")
with patch("celery.worker.consumer.error") as error:
g.on_elect(event)
self.assertTrue(error.called)
def test_on_elect(self):
c = self.Consumer()
g = Gossip(c)
g.start(c)
event = self.Event('id1')
g.on_elect(event)
in_heap = g.consensus_requests['id1']
self.assertTrue(in_heap)
g.dispatcher.send.assert_called_with('worker-elect-ack', id='id1')
event.pop('clock')
with patch('celery.worker.consumer.error') as error:
g.on_elect(event)
self.assertTrue(error.called)
def test_periodic(self):
c = self.Consumer()
g = Gossip(c)
g.on_node_lost = Mock()
state = g.state = Mock()
worker = Mock()
state.workers = {'foo': worker}
worker.alive = True
worker.hostname = 'foo'
g.periodic()
worker.alive = False
g.periodic()
g.on_node_lost.assert_called_with(worker)
with self.assertRaises(KeyError):
state.workers['foo']
def test_call_task(self):
c = self.Consumer()
g = Gossip(c)
g.start(c)
with patch('celery.worker.consumer.signature') as signature:
sig = signature.return_value = Mock()
task = Mock()
g.call_task(task)
signature.assert_called_with(task, app=c.app)
sig.apply_async.assert_called_with()
sig.apply_async.side_effect = MemoryError()
with patch('celery.worker.consumer.error') as error:
g.call_task(task)
self.assertTrue(error.called)
def test_callbacks(self):
c = self.Consumer()
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
on_node_join = Mock(name='on_node_join')
on_node_join2 = Mock(name='on_node_join2')
on_node_leave = Mock(name='on_node_leave')
on_node_lost = Mock(name='on.node_lost')
g.on.node_join.add(on_node_join)
g.on.node_join.add(on_node_join2)
g.on.node_leave.add(on_node_leave)
g.on.node_lost.add(on_node_lost)
worker = Mock(name='worker')
g.on_node_join(worker)
on_node_join.assert_called_with(worker)
on_node_join2.assert_called_with(worker)
g.on_node_leave(worker)
on_node_leave.assert_called_with(worker)
g.on_node_lost(worker)
on_node_lost.assert_called_with(worker)
def test_periodic(self):
c = self.Consumer()
g = Gossip(c)
g.on_node_lost = Mock()
state = g.state = Mock()
worker = Mock()
state.workers = {'foo': worker}
worker.alive = True
worker.hostname = 'foo'
g.periodic()
worker.alive = False
g.periodic()
g.on_node_lost.assert_called_with(worker)
with self.assertRaises(KeyError):
state.workers['foo']
def test_periodic(self):
c = self.Consumer()
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
g.on_node_lost = Mock()
state = g.state = Mock()
worker = Mock()
state.workers = {"foo": worker}
worker.alive = True
worker.hostname = "foo"
g.periodic()
worker.alive = False
g.periodic()
g.on_node_lost.assert_called_with(worker)
with self.assertRaises(KeyError):
state.workers["foo"]
def test_callbacks(self):
c = self.Consumer()
c.app.connection = _amqp_connection()
g = Gossip(c)
on_node_join = Mock(name='on_node_join')
on_node_join2 = Mock(name='on_node_join2')
on_node_leave = Mock(name='on_node_leave')
on_node_lost = Mock(name='on.node_lost')
g.on.node_join.add(on_node_join)
g.on.node_join.add(on_node_join2)
g.on.node_leave.add(on_node_leave)
g.on.node_lost.add(on_node_lost)
worker = Mock(name='worker')
g.on_node_join(worker)
on_node_join.assert_called_with(worker)
on_node_join2.assert_called_with(worker)
g.on_node_leave(worker)
on_node_leave.assert_called_with(worker)
g.on_node_lost(worker)
on_node_lost.assert_called_with(worker)
def test_on_node_leave(self):
c = self.Consumer()
g = Gossip(c)
with patch('celery.worker.consumer.info') as info:
g.on_node_leave(c)
info.assert_called_with('%s left', '*****@*****.**')
def test_on_node_leave(self):
c = self.Consumer()
g = Gossip(c)
with patch("celery.worker.consumer.info") as info:
g.on_node_leave(c)
info.assert_called_with("%s left", "*****@*****.**")
def test_on_node_lost(self):
c = self.Consumer()
g = Gossip(c)
with patch('celery.worker.consumer.info') as info:
g.on_node_lost(c)
info.assert_called_with('missed heartbeat from %s', '*****@*****.**')
def test_on_node_leave(self):
c = self.Consumer()
g = Gossip(c)
with patch('celery.worker.consumer.debug') as debug:
g.on_node_leave(c)
debug.assert_called_with('%s left', '*****@*****.**')
def test_on_message(self):
c = self.Consumer()
g = Gossip(c)
prepare = Mock()
prepare.return_value = "worker-online", {}
g.update_state = Mock()
worker = Mock()
g.on_node_join = Mock()
g.on_node_leave = Mock()
g.update_state.return_value = worker, 1
message = Mock()
message.delivery_info = {"routing_key": "worker-online"}
message.headers = {"hostname": "other"}
handler = g.event_handlers["worker-online"] = Mock()
g.on_message(prepare, message)
handler.assert_called_with(message.payload)
g.event_handlers = {}
g.on_message(prepare, message)
g.on_node_join.assert_called_with(worker)
message.delivery_info = {"routing_key": "worker-offline"}
prepare.return_value = "worker-offline", {}
g.on_message(prepare, message)
g.on_node_leave.assert_called_with(worker)
message.delivery_info = {"routing_key": "worker-baz"}
prepare.return_value = "worker-baz", {}
g.update_state.return_value = worker, 0
g.on_message(prepare, message)
g.on_node_leave.reset_mock()
message.headers = {"hostname": g.hostname}
g.on_message(prepare, message)
self.assertFalse(g.on_node_leave.called)
g.clock.forward.assert_called_with()
def test_on_node_lost(self):
c = self.Consumer()
g = Gossip(c)
with patch('celery.worker.consumer.warn') as warn:
g.on_node_lost(c)
warn.assert_called_with('%s went missing!', '*****@*****.**')
def test_on_node_join(self):
c = self.Consumer()
g = Gossip(c)
with patch('celery.worker.consumer.info') as info:
g.on_node_join(c)
info.assert_called_with('%s joined the party', '*****@*****.**')
def test_on_message(self):
c = self.Consumer()
c.app.connection_for_read = _amqp_connection()
g = Gossip(c)
self.assertTrue(g.enabled)
prepare = Mock()
prepare.return_value = "worker-online", {}
c.app.events.State.assert_called_with(
on_node_join=g.on_node_join, on_node_leave=g.on_node_leave, max_tasks_in_memory=1
)
g.update_state = Mock()
worker = Mock()
g.on_node_join = Mock()
g.on_node_leave = Mock()
g.update_state.return_value = worker, 1
message = Mock()
message.delivery_info = {"routing_key": "worker-online"}
message.headers = {"hostname": "other"}
handler = g.event_handlers["worker-online"] = Mock()
g.on_message(prepare, message)
handler.assert_called_with(message.payload)
g.event_handlers = {}
g.on_message(prepare, message)
message.delivery_info = {"routing_key": "worker-offline"}
prepare.return_value = "worker-offline", {}
g.on_message(prepare, message)
message.delivery_info = {"routing_key": "worker-baz"}
prepare.return_value = "worker-baz", {}
g.update_state.return_value = worker, 0
g.on_message(prepare, message)
message.headers = {"hostname": g.hostname}
g.on_message(prepare, message)
g.clock.forward.assert_called_with()
def test_init(self):
c = self.Consumer()
g = Gossip(c)
self.assertTrue(g.enabled)
self.assertIs(c.gossip, g)
def test_on_message(self):
c = self.Consumer()
g = Gossip(c)
self.assertTrue(g.enabled)
prepare = Mock()
prepare.return_value = 'worker-online', {}
c.app.events.State.assert_called_with(
on_node_join=g.on_node_join,
on_node_leave=g.on_node_leave,
max_tasks_in_memory=1,
)
g.update_state = Mock()
worker = Mock()
g.on_node_join = Mock()
g.on_node_leave = Mock()
g.update_state.return_value = worker, 1
message = Mock()
message.delivery_info = {'routing_key': 'worker-online'}
message.headers = {'hostname': 'other'}
handler = g.event_handlers['worker-online'] = Mock()
g.on_message(prepare, message)
handler.assert_called_with(message.payload)
g.event_handlers = {}
g.on_message(prepare, message)
message.delivery_info = {'routing_key': 'worker-offline'}
prepare.return_value = 'worker-offline', {}
g.on_message(prepare, message)
message.delivery_info = {'routing_key': 'worker-baz'}
prepare.return_value = 'worker-baz', {}
g.update_state.return_value = worker, 0
g.on_message(prepare, message)
message.headers = {'hostname': g.hostname}
g.on_message(prepare, message)
g.clock.forward.assert_called_with()
def test_init(self):
c = self.Consumer()
c.app.connection = _amqp_connection()
g = Gossip(c)
self.assertTrue(g.enabled)
self.assertIs(c.gossip, g)
def test_on_node_join(self):
c = self.Consumer()
g = Gossip(c)
with patch('celery.worker.consumer.debug') as debug:
g.on_node_join(c)
debug.assert_called_with('%s joined the party', '*****@*****.**')
def test_on_message(self):
c = self.Consumer()
g = Gossip(c)
self.assertTrue(g.enabled)
prepare = Mock()
prepare.return_value = 'worker-online', {}
c.app.events.State.assert_called_with(
on_node_join=g.on_node_join,
on_node_leave=g.on_node_leave,
max_tasks_in_memory=1,
)
g.update_state = Mock()
worker = Mock()
g.on_node_join = Mock()
g.on_node_leave = Mock()
g.update_state.return_value = worker, 1
message = Mock()
message.delivery_info = {'routing_key': 'worker-online'}
message.headers = {'hostname': 'other'}
handler = g.event_handlers['worker-online'] = Mock()
g.on_message(prepare, message)
handler.assert_called_with(message.payload)
g.event_handlers = {}
g.on_message(prepare, message)
message.delivery_info = {'routing_key': 'worker-offline'}
prepare.return_value = 'worker-offline', {}
g.on_message(prepare, message)
message.delivery_info = {'routing_key': 'worker-baz'}
prepare.return_value = 'worker-baz', {}
g.update_state.return_value = worker, 0
g.on_message(prepare, message)
message.headers = {'hostname': g.hostname}
g.on_message(prepare, message)
g.clock.forward.assert_called_with()
def test_on_node_lost(self):
c = self.Consumer()
g = Gossip(c)
with patch('celery.worker.consumer.warn') as warn:
g.on_node_lost(c)
warn.assert_called_with('%s went missing!', '*****@*****.**')
