def test_task_complete_call_back(self):
# Test the workflow of the task complete callback.
block_size = 1
concurrent_tasks = 2
# Fake task arg generator:
hc_tag = iter(xrange(4))
self.calc.progress['total'] = 7
self.calc.progress['hc_total'] = 4
callback = self.calc.get_task_complete_callback(
hc_tag, block_size=block_size, concurrent_tasks=concurrent_tasks)
message = self.__class__.FakeMessage()
# "pre-queue" two hazard curve tasks,
# and use a fake function
base.queue_next(lambda x: x, hc_tag.next())
base.queue_next(lambda x: x, hc_tag.next())
self.assertEqual(2, self.queue_next_mock.call_count)
self.calc.progress['in_queue'] = 2
# message body:
body = dict(job_id=self.job.id)
# First call:
body['num_items'] = 1
body['calc_type'] = 'hazard_curve'
callback(body, message)
self.assertEqual(1, message.acks)
self.assertFalse(self.calc.disagg_phase)
self.assertEqual(
dict(total=7, computed=1, hc_total=4, hc_computed=1, in_queue=2),
self.calc.progress)
self.assertEqual(3, self.queue_next_mock.call_count)
self.assertEqual(0, self.finalize_curves_mock.call_count)
# Second call:
callback(body, message)
self.assertEqual(2, message.acks)
self.assertFalse(self.calc.disagg_phase)
self.assertEqual(
dict(total=7, computed=2, hc_total=4, hc_computed=2, in_queue=2),
self.calc.progress)
self.assertEqual(4, self.queue_next_mock.call_count)
self.assertEqual(0, self.finalize_curves_mock.call_count)
# Test that an exception is thrown when we receive a non-hazard_curve
# completion message during the hazard curve phase.
# This exception case is meant to test for invalid calculator workflow.
body['calc_type'] = 'disagg' # could be any fake value as well
self.assertRaises(RuntimeError, callback, body, message)
# Third call:
body['calc_type'] = 'hazard_curve'
callback(body, message)
self.assertEqual(3, message.acks)
self.assertFalse(self.calc.disagg_phase)
self.assertEqual(
# There is one hazard curve task left in the queue.
dict(total=7, computed=3, hc_total=4, hc_computed=3, in_queue=1),
self.calc.progress)
self.assertEqual(4, self.queue_next_mock.call_count)
# Fourth call (the last hazard curve task):
body['calc_type'] = 'hazard_curve'
callback(body, message)
self.assertEqual(4, message.acks)
# Hazard curves are done, so here we should switch to the disagg phase
self.assertTrue(self.calc.disagg_phase)
self.assertEqual(
dict(total=7, computed=4, hc_total=4, hc_computed=4, in_queue=2),
self.calc.progress)
# We should have queued 2 disagg tasks here (given concurrent_tasks=2)
self.assertEqual(6, self.queue_next_mock.call_count)
self.assertEqual(1, self.finalize_curves_mock.call_count)
# Fourth call:
body['calc_type'] = 'disagg'
callback(body, message)
self.assertEqual(5, message.acks)
self.assertTrue(self.calc.disagg_phase)
self.assertEqual(
dict(total=7, computed=5, hc_total=4, hc_computed=4, in_queue=2),
self.calc.progress)
self.assertEqual(7, self.queue_next_mock.call_count)
self.assertEqual(1, self.finalize_curves_mock.call_count)
# Fifth call:
callback(body, message)
self.assertEqual(6, message.acks)
self.assertTrue(self.calc.disagg_phase)
self.assertEqual(
dict(total=7, computed=6, hc_total=4, hc_computed=4, in_queue=1),
self.calc.progress)
# Nothing else should be queued; there are no more items to enque.
self.assertEqual(7, self.queue_next_mock.call_count)
self.assertEqual(1, self.finalize_curves_mock.call_count)
# Sixth (final) call:
# This simulates the message from the last task. The only expected
# effects here are:
# - message ack
# - updated 'computed' counter
callback(body, message)
self.assertEqual(7, message.acks)
# Hazard curves computed counter remains at 3
self.assertEqual(
dict(total=7, computed=7, hc_total=4, hc_computed=4, in_queue=0),
self.calc.progress)
self.assertEqual(1, self.finalize_curves_mock.call_count)
def test_task_complete_call_back(self):
# Test the workflow of the task complete callback.
block_size = 1
concurrent_tasks = 2
# Fake task arg generator:
hc_tag = iter(xrange(4))
self.calc.progress['total'] = 7
self.calc.progress['hc_total'] = 4
callback = self.calc.get_task_complete_callback(
hc_tag, block_size=block_size, concurrent_tasks=concurrent_tasks)
message = self.__class__.FakeMessage()
# "pre-queue" two hazard curve tasks,
# and use a fake function
base.queue_next(lambda x: x, hc_tag.next())
base.queue_next(lambda x: x, hc_tag.next())
self.assertEqual(2, self.queue_next_mock.call_count)
self.calc.progress['in_queue'] = 2
# message body:
body = dict(job_id=self.job.id)
# First call:
body['num_items'] = 1
body['calc_type'] = 'hazard_curve'
callback(body, message)
self.assertEqual(1, message.acks)
self.assertFalse(self.calc.disagg_phase)
self.assertEqual(
dict(total=7, computed=1, hc_total=4, hc_computed=1, in_queue=2),
self.calc.progress)
self.assertEqual(3, self.queue_next_mock.call_count)
self.assertEqual(0, self.finalize_curves_mock.call_count)
# Second call:
callback(body, message)
self.assertEqual(2, message.acks)
self.assertFalse(self.calc.disagg_phase)
self.assertEqual(
dict(total=7, computed=2, hc_total=4, hc_computed=2, in_queue=2),
self.calc.progress)
self.assertEqual(4, self.queue_next_mock.call_count)
self.assertEqual(0, self.finalize_curves_mock.call_count)
# Test that an exception is thrown when we receive a non-hazard_curve
# completion message during the hazard curve phase.
# This exception case is meant to test for invalid calculator workflow.
body['calc_type'] = 'disagg' # could be any fake value as well
self.assertRaises(RuntimeError, callback, body, message)
# Third call:
body['calc_type'] = 'hazard_curve'
callback(body, message)
self.assertEqual(3, message.acks)
self.assertFalse(self.calc.disagg_phase)
self.assertEqual(
# There is one hazard curve task left in the queue.
dict(total=7, computed=3, hc_total=4, hc_computed=3, in_queue=1),
self.calc.progress)
self.assertEqual(4, self.queue_next_mock.call_count)
# Fourth call (the last hazard curve task):
body['calc_type'] = 'hazard_curve'
callback(body, message)
self.assertEqual(4, message.acks)
# Hazard curves are done, so here we should switch to the disagg phase
self.assertTrue(self.calc.disagg_phase)
self.assertEqual(
dict(total=7, computed=4, hc_total=4, hc_computed=4, in_queue=2),
self.calc.progress)
# We should have queued 2 disagg tasks here (given concurrent_tasks=2)
self.assertEqual(6, self.queue_next_mock.call_count)
self.assertEqual(1, self.finalize_curves_mock.call_count)
# Fourth call:
body['calc_type'] = 'disagg'
callback(body, message)
self.assertEqual(5, message.acks)
self.assertTrue(self.calc.disagg_phase)
self.assertEqual(
dict(total=7, computed=5, hc_total=4, hc_computed=4, in_queue=2),
self.calc.progress)
self.assertEqual(7, self.queue_next_mock.call_count)
self.assertEqual(1, self.finalize_curves_mock.call_count)
# Fifth call:
callback(body, message)
self.assertEqual(6, message.acks)
self.assertTrue(self.calc.disagg_phase)
self.assertEqual(
dict(total=7, computed=6, hc_total=4, hc_computed=4, in_queue=1),
self.calc.progress)
# Nothing else should be queued; there are no more items to enque.
self.assertEqual(7, self.queue_next_mock.call_count)
self.assertEqual(1, self.finalize_curves_mock.call_count)
# Sixth (final) call:
# This simulates the message from the last task. The only expected
# effects here are:
# - message ack
# - updated 'computed' counter
callback(body, message)
self.assertEqual(7, message.acks)
# Hazard curves computed counter remains at 3
self.assertEqual(
dict(total=7, computed=7, hc_total=4, hc_computed=4, in_queue=0),
self.calc.progress)
self.assertEqual(1, self.finalize_curves_mock.call_count)
def callback(body, message):
"""
:param dict body:
``body`` is the message sent by the task. The dict should
contain 2 keys: `job_id` and `num_sources` (to indicate the
number of sources computed).
Both values are `int`.
:param message:
A :class:`kombu.transport.pyamqplib.Message`, which contains
metadata about the message (including content type, channel,
etc.). See kombu docs for more details.
"""
job_id = body['job_id']
num_items = body['num_items']
calc_type = body['calc_type']
assert job_id == self.job.id
# Log a progress message
logs.log_percent_complete(job_id, 'hazard')
if self.disagg_phase:
assert calc_type == 'disagg'
# We're in the second phase of the calculation; just keep
# queuing tasks (if there are any left) and wait for everything
# to finish.
try:
base.queue_next(
self.core_calc_task, disagg_task_arg_gen.next())
except StopIteration:
# There are no more tasks to dispatch; now we just need to
# wait until all of the tasks signal completion.
self.progress['in_queue'] -= 1
else:
logs.LOG.debug('* queuing the next disagg task')
else:
if calc_type == 'hazard_curve':
# record progress specifically for hazard curve computation
self.progress['hc_computed'] += num_items
if (self.progress['hc_computed']
== self.progress['hc_total']):
# we just finished the last hazard curve task ...
self.progress['in_queue'] -= 1
# ... and we're switching to disagg phase
self.disagg_phase = True
logs.LOG.progress('Hazard curve computation complete',
indent=True)
logs.LOG.progress('Starting disaggregation',
indent=True)
# Finalize the hazard curves, so the disaggregation
# can find curves by their point geometry:
self.finalize_hazard_curves()
logs.LOG.debug('* queuing initial disagg tasks')
# the task queue should be empty, so let's fill it up
# with disagg tasks:
for _ in xrange(concurrent_tasks):
try:
base.queue_next(
self.core_calc_task,
disagg_task_arg_gen.next())
except StopIteration:
# If we get a `StopIteration` here, that means
# we have number of disagg tasks <
# concurrent_tasks.
break
else:
self.progress['in_queue'] += 1
logs.LOG.info('Tasks now in queue: %s'
% self.progress['in_queue'])
else:
# we're not done computing hazard curves; enqueue the
# next task
try:
base.queue_next(
self.core_calc_task, hc_task_arg_gen.next())
except StopIteration:
# No more hazard curve tasks left to enqueue;
# now we just wait for this phase to complete.
self.progress['in_queue'] -= 1
else:
logs.LOG.debug(
'* queueing the next hazard curve task')
else:
# we're in the hazard curve phase, but the completed
# message did not have a 'hazard_curve' type
raise RuntimeError(
'Unexpected message `calc_type`: "%s"' % calc_type)
# Last thing, update the 'computed' counter and acknowledge the
# message:
self.progress['computed'] += num_items
message.ack()
logs.LOG.info('A task was completed. Tasks now in queue: %s'
% self.progress['in_queue'])
def callback(body, message):
"""
:param dict body:
``body`` is the message sent by the task. The dict should
contain 2 keys: `job_id` and `num_sources` (to indicate the
number of sources computed).
Both values are `int`.
:param message:
A :class:`kombu.transport.pyamqplib.Message`, which contains
metadata about the message (including content type, channel,
etc.). See kombu docs for more details.
"""
job_id = body['job_id']
num_items = body['num_items']
calc_type = body['calc_type']
assert job_id == self.job.id
# Log a progress message
logs.log_percent_complete(job_id, 'hazard')
if self.disagg_phase:
assert calc_type == 'disagg'
# We're in the second phase of the calculation; just keep
# queuing tasks (if there are any left) and wait for everything
# to finish.
try:
base.queue_next(self.core_calc_task,
disagg_task_arg_gen.next())
except StopIteration:
# There are no more tasks to dispatch; now we just need to
# wait until all of the tasks signal completion.
self.progress['in_queue'] -= 1
else:
logs.LOG.debug('* queuing the next disagg task')
else:
if calc_type == 'hazard_curve':
# record progress specifically for hazard curve computation
self.progress['hc_computed'] += num_items
if (self.progress['hc_computed'] ==
self.progress['hc_total']):
# we just finished the last hazard curve task ...
self.progress['in_queue'] -= 1
# ... and we're switching to disagg phase
self.disagg_phase = True
logs.LOG.progress('Hazard curve computation complete',
indent=True)
logs.LOG.progress('Starting disaggregation',
indent=True)
# Finalize the hazard curves, so the disaggregation
# can find curves by their point geometry:
self.finalize_hazard_curves()
logs.LOG.debug('* queuing initial disagg tasks')
# the task queue should be empty, so let's fill it up
# with disagg tasks:
for _ in xrange(concurrent_tasks):
try:
base.queue_next(self.core_calc_task,
disagg_task_arg_gen.next())
except StopIteration:
# If we get a `StopIteration` here, that means
# we have number of disagg tasks <
# concurrent_tasks.
break
else:
self.progress['in_queue'] += 1
logs.LOG.info('Tasks now in queue: %s' %
self.progress['in_queue'])
else:
# we're not done computing hazard curves; enqueue the
# next task
try:
base.queue_next(self.core_calc_task,
hc_task_arg_gen.next())
except StopIteration:
# No more hazard curve tasks left to enqueue;
# now we just wait for this phase to complete.
self.progress['in_queue'] -= 1
else:
logs.LOG.debug(
'* queueing the next hazard curve task')
else:
# we're in the hazard curve phase, but the completed
# message did not have a 'hazard_curve' type
raise RuntimeError('Unexpected message `calc_type`: "%s"' %
calc_type)
# Last thing, update the 'computed' counter and acknowledge the
# message:
self.progress['computed'] += num_items
message.ack()
logs.LOG.info('A task was completed. Tasks now in queue: %s' %
self.progress['in_queue'])
