def test_start_job_malformed(logger, callback): mock_consumer.__iter__ = lambda x: iter([rec34]) worker = Worker( hosts='localhost', topic='foo', timeout=1000, callback=callback, ) worker.start() logger.info.assert_has_calls([ mock.call('Starting Worker(topic=foo) ...'), mock.call('Processing {} ...'.format(rec34_repr)), ]) logger.warning.assert_called_once_with( '{} malformed. Skipping ...'.format(rec34_repr)) assert not callback.called
def test_start_job_success(logger, callback): mock_consumer.__iter__ = lambda x: iter([rec11]) worker = Worker( hosts='localhost', topic='foo', callback=callback, ) worker.start() logger.info.assert_has_calls([ mock.call('Starting Worker(topic=foo) ...'), mock.call('Processing {} ...'.format(rec11_repr)), mock.call('Running Job 100: tests.utils.success_func(1, 2, c=3) ...'), mock.call('Job 100 returned: (1, 2, 3)'), mock.call('Executing callback ...') ]) callback.assert_called_once_with('success', success_job, (1, 2, 3), None, None)
def test_start_job_timeout(logger, callback): mock_consumer.__iter__ = lambda x: iter([rec21]) worker = Worker( hosts='localhost', topic='foo', timeout=1000, callback=callback, ) worker.start() logger.info.assert_has_calls([ mock.call('Starting Worker(topic=foo) ...'), mock.call('Processing {} ...'.format(rec21_repr)), mock.call('Running Job 300: tests.utils.timeout_func(2, 3, 4) ...'), mock.call('Executing callback ...') ]) logger.error.assert_called_once_with( 'Job 300 timed out after 100 seconds.') callback.assert_called_once_with('timeout', timeout_job, None, None, None)
def test_start_job_callback_fail(logger, callback): mock_consumer.__iter__ = lambda x: iter([rec11]) expected_error = KeyError('foo') callback.side_effect = expected_error worker = Worker( hosts='localhost', topic='foo', callback=callback, ) worker.start() logger.info.assert_has_calls([ mock.call('Starting Worker(topic=foo) ...'), mock.call('Processing {} ...'.format(rec11_repr)), mock.call('Running Job 100: tests.utils.success_func(1, 2, c=3) ...'), mock.call('Job 100 returned: (1, 2, 3)'), mock.call('Executing callback ...') ]) logger.exception.assert_called_once_with( 'Callback failed: {}'.format(expected_error))
def test_start_job_failure(logger, callback): mock_consumer.__iter__ = lambda x: iter([rec12]) worker = Worker( hosts='localhost', topic='foo', timeout=1000, callback=callback, ) worker.start() logger.info.assert_has_calls([ mock.call('Starting Worker(topic=foo) ...'), mock.call('Processing {} ...'.format(rec12_repr)), mock.call('Running Job 200: tests.utils.failure_func(1, 2, 3) ...'), mock.call('Executing callback ...') ]) logger.exception.assert_called_with('Job 200 failed: failed!') assert len(callback.call_args_list) == 1 callback_args = callback.call_args_list[0][0] assert callback_args[0] == 'failure' assert callback_args[1] == failure_job assert callback_args[2] is None assert isinstance(callback_args[3], ValueError) assert isinstance(callback_args[4], str)
import logging from kafka import KafkaConsumer from kq import Worker # Set up logging. formatter = logging.Formatter('[%(levelname)s] %(message)s') stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) logger = logging.getLogger('kq.worker') logger.setLevel(logging.DEBUG) logger.addHandler(stream_handler) # Set up a Kafka consumer. consumer = KafkaConsumer(bootstrap_servers='34.73.142.27:9092', group_id='group', auto_offset_reset='latest') # Set up a worker. worker = Worker(topic='topic_queue', consumer=consumer) worker.start()