def test_revert_task(self):
self.message_mock.properties['type'] = pr.NOTIFY
notify = pr.Notify(topic=self.executor_topic, tasks=[self.task.name])
ex = self.executor()
ex._process_notify(notify.to_dict(), self.message_mock)
ex.revert_task(self.task, self.task_uuid, self.task_args,
self.task_result, self.task_failures)
expected_calls = [
mock.call.Request(self.task,
self.task_uuid,
'revert',
self.task_args,
None,
self.timeout,
failures=self.task_failures,
result=self.task_result),
mock.call.request.transition_and_log_error(pr.PENDING,
logger=mock.ANY),
mock.call.proxy.publish(msg=self.request_inst_mock,
routing_key=self.executor_topic,
reply_to=self.executor_uuid,
correlation_id=self.task_uuid)
]
self.assertEqual(expected_calls, self.main_mock.mock_calls)
def test_notify(self):
barrier = threading_utils.Event()
on_notify = mock.MagicMock()
on_notify.side_effect = lambda *args, **kwargs: barrier.set()
handlers = {pr.NOTIFY: on_notify}
p = proxy.Proxy(TEST_TOPIC,
TEST_EXCHANGE,
handlers,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
t = threading_utils.daemon_thread(p.start)
t.start()
p.wait()
p.publish(pr.Notify(), TEST_TOPIC)
self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT))
p.stop()
t.join()
self.assertTrue(on_notify.called)
on_notify.assert_called_with({}, mock.ANY)
def maybe_publish(self):
"""Periodically called to publish notify message to each topic.
These messages (especially the responses) are how this find learns
about workers and what tasks they can perform (so that we can then
match workers to tasks to run).
"""
if self._messages_published == 0:
self._proxy.publish(pr.Notify(), self._topics, reply_to=self._uuid)
self._messages_published += 1
self._watch.restart()
else:
if self._watch.expired():
self._proxy.publish(pr.Notify(),
self._topics,
reply_to=self._uuid)
self._messages_published += 1
self._watch.restart()
def _process_response(self, data, message):
"""Process notify message sent from remote side."""
LOG.debug("Started processing notify response message '%s'",
ku.DelayedPretty(message))
response = pr.Notify(**data)
LOG.debug("Extracted notify response '%s'", response)
with self._cond:
worker, new_or_updated = self._add(response.topic, response.tasks)
if new_or_updated:
LOG.debug(
"Updated worker '%s' (%s total workers are"
" currently known)", worker, self._total_workers())
self._cond.notify_all()
if new_or_updated:
self.notifier.notify(self.WORKER_ARRIVED, {'worker': worker})
def process_response(self, data, message):
"""Process notify message sent from remote side."""
LOG.debug("Started processing notify response message '%s'",
ku.DelayedPretty(message))
response = pr.Notify(**data)
LOG.debug("Extracted notify response '%s'", response)
with self._cond:
worker, new_or_updated = self._add(response.topic, response.tasks)
if new_or_updated:
LOG.debug(
"Updated worker '%s' (%s total workers are"
" currently known)", worker, self.total_workers)
self._cond.notify_all()
worker.last_seen = timeutils.now()
self._messages_processed += 1
def _process_notify(self, notify, message):
"""Process notify message and reply back."""
try:
reply_to = message.properties['reply_to']
except KeyError:
LOG.warn("The 'reply_to' message property is missing"
" in received notify message '%s'",
ku.DelayedPretty(message), exc_info=True)
else:
response = pr.Notify(topic=self._topic,
tasks=self._endpoints.keys())
try:
self._proxy.publish(response, routing_key=reply_to)
except Exception:
LOG.critical("Failed to send reply to '%s' with notify"
" response '%s'", reply_to, response,
exc_info=True)
def test_execute_task(self):
self.message_mock.properties['type'] = pr.NOTIFY
notify = pr.Notify(topic=self.executor_topic, tasks=[self.task.name])
ex = self.executor()
ex._process_notify(notify.to_dict(), self.message_mock)
ex.execute_task(self.task, self.task_uuid, self.task_args)
expected_calls = [
mock.call.Request(self.task, self.task_uuid, 'execute',
self.task_args, self.timeout),
mock.call.request.transition_and_log_error(pr.PENDING,
logger=mock.ANY),
mock.call.proxy.publish(self.request_inst_mock,
self.executor_topic,
reply_to=self.executor_uuid,
correlation_id=self.task_uuid)
]
self.assertEqual(expected_calls, self.master_mock.mock_calls)
def test_execute_task_publish_error(self):
self.message_mock.properties['type'] = pr.NOTIFY
self.proxy_inst_mock.publish.side_effect = Exception('Woot!')
notify = pr.Notify(topic=self.executor_topic, tasks=[self.task.name])
ex = self.executor()
ex._process_notify(notify.to_dict(), self.message_mock)
ex.execute_task(self.task, self.task_uuid, self.task_args)
expected_calls = [
mock.call.Request(self.task, self.task_uuid, 'execute',
self.task_args, None, self.timeout),
mock.call.request.transition_and_log_error(pr.PENDING,
logger=mock.ANY),
mock.call.proxy.publish(msg=self.request_inst_mock,
routing_key=self.executor_topic,
reply_to=self.executor_uuid,
correlation_id=self.task_uuid),
mock.call.request.transition_and_log_error(pr.FAILURE,
logger=mock.ANY),
mock.call.request.set_result(mock.ANY)
]
self.assertEqual(expected_calls, self.main_mock.mock_calls)
def test_multi_message(self):
message_count = 30
barrier = latch.Latch(message_count)
countdown = lambda data, message: barrier.countdown()
on_notify = mock.MagicMock()
on_notify.side_effect = countdown
on_response = mock.MagicMock()
on_response.side_effect = countdown
on_request = mock.MagicMock()
on_request.side_effect = countdown
handlers = {
pr.NOTIFY: on_notify,
pr.RESPONSE: on_response,
pr.REQUEST: on_request,
}
p = proxy.Proxy(TEST_TOPIC,
TEST_EXCHANGE,
handlers,
transport='memory',
transport_options={
'polling_interval': POLLING_INTERVAL,
})
t = threading_utils.daemon_thread(p.start)
t.start()
p.wait()
for i in range(0, message_count):
j = i % 3
if j == 0:
p.publish(pr.Notify(), TEST_TOPIC)
elif j == 1:
p.publish(pr.Response(pr.RUNNING), TEST_TOPIC)
else:
p.publish(
pr.Request(test_utils.DummyTask("dummy_%s" % i),
uuidutils.generate_uuid(), pr.EXECUTE, [],
None), TEST_TOPIC)
self.assertTrue(barrier.wait(test_utils.WAIT_TIMEOUT))
self.assertEqual(0, barrier.needed)
p.stop()
t.join()
self.assertTrue(on_notify.called)
self.assertTrue(on_response.called)
self.assertTrue(on_request.called)
self.assertEqual(10, on_notify.call_count)
self.assertEqual(10, on_response.call_count)
self.assertEqual(10, on_request.call_count)
call_count = sum([
on_notify.call_count,
on_response.call_count,
on_request.call_count,
])
self.assertEqual(message_count, call_count)
def test_reply_notify(self):
msg = pr.Notify(topic="bob", tasks=['a', 'b', 'c'])
pr.Notify.validate(msg.to_dict(), True)
def test_send_notify(self):
msg = pr.Notify()
pr.Notify.validate(msg.to_dict(), False)
def _notify_topics(self):
"""Cyclically called to publish notify message to each topic."""
self._proxy.publish(pr.Notify(), self._topics, reply_to=self._uuid)
