def test_task_complete_call_back(self): # Test the workflow of the task complete callback. block_size = 1 concurrent_tasks = 2 # Fake task arg generator: hc_tag = iter(xrange(4)) self.calc.progress['total'] = 7 self.calc.progress['hc_total'] = 4 callback = self.calc.get_task_complete_callback( hc_tag, block_size=block_size, concurrent_tasks=concurrent_tasks) message = self.__class__.FakeMessage() # "pre-queue" two hazard curve tasks, # and use a fake function base.queue_next(lambda x: x, hc_tag.next()) base.queue_next(lambda x: x, hc_tag.next()) self.assertEqual(2, self.queue_next_mock.call_count) self.calc.progress['in_queue'] = 2 # message body: body = dict(job_id=self.job.id) # First call: body['num_items'] = 1 body['calc_type'] = 'hazard_curve' callback(body, message) self.assertEqual(1, message.acks) self.assertFalse(self.calc.disagg_phase) self.assertEqual( dict(total=7, computed=1, hc_total=4, hc_computed=1, in_queue=2), self.calc.progress) self.assertEqual(3, self.queue_next_mock.call_count) self.assertEqual(0, self.finalize_curves_mock.call_count) # Second call: callback(body, message) self.assertEqual(2, message.acks) self.assertFalse(self.calc.disagg_phase) self.assertEqual( dict(total=7, computed=2, hc_total=4, hc_computed=2, in_queue=2), self.calc.progress) self.assertEqual(4, self.queue_next_mock.call_count) self.assertEqual(0, self.finalize_curves_mock.call_count) # Test that an exception is thrown when we receive a non-hazard_curve # completion message during the hazard curve phase. # This exception case is meant to test for invalid calculator workflow. body['calc_type'] = 'disagg' # could be any fake value as well self.assertRaises(RuntimeError, callback, body, message) # Third call: body['calc_type'] = 'hazard_curve' callback(body, message) self.assertEqual(3, message.acks) self.assertFalse(self.calc.disagg_phase) self.assertEqual( # There is one hazard curve task left in the queue. dict(total=7, computed=3, hc_total=4, hc_computed=3, in_queue=1), self.calc.progress) self.assertEqual(4, self.queue_next_mock.call_count) # Fourth call (the last hazard curve task): body['calc_type'] = 'hazard_curve' callback(body, message) self.assertEqual(4, message.acks) # Hazard curves are done, so here we should switch to the disagg phase self.assertTrue(self.calc.disagg_phase) self.assertEqual( dict(total=7, computed=4, hc_total=4, hc_computed=4, in_queue=2), self.calc.progress) # We should have queued 2 disagg tasks here (given concurrent_tasks=2) self.assertEqual(6, self.queue_next_mock.call_count) self.assertEqual(1, self.finalize_curves_mock.call_count) # Fourth call: body['calc_type'] = 'disagg' callback(body, message) self.assertEqual(5, message.acks) self.assertTrue(self.calc.disagg_phase) self.assertEqual( dict(total=7, computed=5, hc_total=4, hc_computed=4, in_queue=2), self.calc.progress) self.assertEqual(7, self.queue_next_mock.call_count) self.assertEqual(1, self.finalize_curves_mock.call_count) # Fifth call: callback(body, message) self.assertEqual(6, message.acks) self.assertTrue(self.calc.disagg_phase) self.assertEqual( dict(total=7, computed=6, hc_total=4, hc_computed=4, in_queue=1), self.calc.progress) # Nothing else should be queued; there are no more items to enque. self.assertEqual(7, self.queue_next_mock.call_count) self.assertEqual(1, self.finalize_curves_mock.call_count) # Sixth (final) call: # This simulates the message from the last task. The only expected # effects here are: # - message ack # - updated 'computed' counter callback(body, message) self.assertEqual(7, message.acks) # Hazard curves computed counter remains at 3 self.assertEqual( dict(total=7, computed=7, hc_total=4, hc_computed=4, in_queue=0), self.calc.progress) self.assertEqual(1, self.finalize_curves_mock.call_count)
def callback(body, message): """ :param dict body: ``body`` is the message sent by the task. The dict should contain 2 keys: `job_id` and `num_sources` (to indicate the number of sources computed). Both values are `int`. :param message: A :class:`kombu.transport.pyamqplib.Message`, which contains metadata about the message (including content type, channel, etc.). See kombu docs for more details. """ job_id = body['job_id'] num_items = body['num_items'] calc_type = body['calc_type'] assert job_id == self.job.id # Log a progress message logs.log_percent_complete(job_id, 'hazard') if self.disagg_phase: assert calc_type == 'disagg' # We're in the second phase of the calculation; just keep # queuing tasks (if there are any left) and wait for everything # to finish. try: base.queue_next( self.core_calc_task, disagg_task_arg_gen.next()) except StopIteration: # There are no more tasks to dispatch; now we just need to # wait until all of the tasks signal completion. self.progress['in_queue'] -= 1 else: logs.LOG.debug('* queuing the next disagg task') else: if calc_type == 'hazard_curve': # record progress specifically for hazard curve computation self.progress['hc_computed'] += num_items if (self.progress['hc_computed'] == self.progress['hc_total']): # we just finished the last hazard curve task ... self.progress['in_queue'] -= 1 # ... and we're switching to disagg phase self.disagg_phase = True logs.LOG.progress('Hazard curve computation complete', indent=True) logs.LOG.progress('Starting disaggregation', indent=True) # Finalize the hazard curves, so the disaggregation # can find curves by their point geometry: self.finalize_hazard_curves() logs.LOG.debug('* queuing initial disagg tasks') # the task queue should be empty, so let's fill it up # with disagg tasks: for _ in xrange(concurrent_tasks): try: base.queue_next( self.core_calc_task, disagg_task_arg_gen.next()) except StopIteration: # If we get a `StopIteration` here, that means # we have number of disagg tasks < # concurrent_tasks. break else: self.progress['in_queue'] += 1 logs.LOG.info('Tasks now in queue: %s' % self.progress['in_queue']) else: # we're not done computing hazard curves; enqueue the # next task try: base.queue_next( self.core_calc_task, hc_task_arg_gen.next()) except StopIteration: # No more hazard curve tasks left to enqueue; # now we just wait for this phase to complete. self.progress['in_queue'] -= 1 else: logs.LOG.debug( '* queueing the next hazard curve task') else: # we're in the hazard curve phase, but the completed # message did not have a 'hazard_curve' type raise RuntimeError( 'Unexpected message `calc_type`: "%s"' % calc_type) # Last thing, update the 'computed' counter and acknowledge the # message: self.progress['computed'] += num_items message.ack() logs.LOG.info('A task was completed. Tasks now in queue: %s' % self.progress['in_queue'])
def callback(body, message): """ :param dict body: ``body`` is the message sent by the task. The dict should contain 2 keys: `job_id` and `num_sources` (to indicate the number of sources computed). Both values are `int`. :param message: A :class:`kombu.transport.pyamqplib.Message`, which contains metadata about the message (including content type, channel, etc.). See kombu docs for more details. """ job_id = body['job_id'] num_items = body['num_items'] calc_type = body['calc_type'] assert job_id == self.job.id # Log a progress message logs.log_percent_complete(job_id, 'hazard') if self.disagg_phase: assert calc_type == 'disagg' # We're in the second phase of the calculation; just keep # queuing tasks (if there are any left) and wait for everything # to finish. try: base.queue_next(self.core_calc_task, disagg_task_arg_gen.next()) except StopIteration: # There are no more tasks to dispatch; now we just need to # wait until all of the tasks signal completion. self.progress['in_queue'] -= 1 else: logs.LOG.debug('* queuing the next disagg task') else: if calc_type == 'hazard_curve': # record progress specifically for hazard curve computation self.progress['hc_computed'] += num_items if (self.progress['hc_computed'] == self.progress['hc_total']): # we just finished the last hazard curve task ... self.progress['in_queue'] -= 1 # ... and we're switching to disagg phase self.disagg_phase = True logs.LOG.progress('Hazard curve computation complete', indent=True) logs.LOG.progress('Starting disaggregation', indent=True) # Finalize the hazard curves, so the disaggregation # can find curves by their point geometry: self.finalize_hazard_curves() logs.LOG.debug('* queuing initial disagg tasks') # the task queue should be empty, so let's fill it up # with disagg tasks: for _ in xrange(concurrent_tasks): try: base.queue_next(self.core_calc_task, disagg_task_arg_gen.next()) except StopIteration: # If we get a `StopIteration` here, that means # we have number of disagg tasks < # concurrent_tasks. break else: self.progress['in_queue'] += 1 logs.LOG.info('Tasks now in queue: %s' % self.progress['in_queue']) else: # we're not done computing hazard curves; enqueue the # next task try: base.queue_next(self.core_calc_task, hc_task_arg_gen.next()) except StopIteration: # No more hazard curve tasks left to enqueue; # now we just wait for this phase to complete. self.progress['in_queue'] -= 1 else: logs.LOG.debug( '* queueing the next hazard curve task') else: # we're in the hazard curve phase, but the completed # message did not have a 'hazard_curve' type raise RuntimeError('Unexpected message `calc_type`: "%s"' % calc_type) # Last thing, update the 'computed' counter and acknowledge the # message: self.progress['computed'] += num_items message.ack() logs.LOG.info('A task was completed. Tasks now in queue: %s' % self.progress['in_queue'])