def test_start_job_malformed(logger, callback):
mock_consumer.__iter__ = lambda x: iter([rec34])
worker = Worker(
hosts='localhost',
topic='foo',
timeout=1000,
callback=callback,
)
worker.start()
logger.info.assert_has_calls([
mock.call('Starting Worker(topic=foo) ...'),
mock.call('Processing {} ...'.format(rec34_repr)),
])
logger.warning.assert_called_once_with(
'{} malformed. Skipping ...'.format(rec34_repr))
assert not callback.called
def test_start_job_success(logger, callback):
mock_consumer.__iter__ = lambda x: iter([rec11])
worker = Worker(
hosts='localhost',
topic='foo',
callback=callback,
)
worker.start()
logger.info.assert_has_calls([
mock.call('Starting Worker(topic=foo) ...'),
mock.call('Processing {} ...'.format(rec11_repr)),
mock.call('Running Job 100: tests.utils.success_func(1, 2, c=3) ...'),
mock.call('Job 100 returned: (1, 2, 3)'),
mock.call('Executing callback ...')
])
callback.assert_called_once_with('success', success_job, (1, 2, 3), None,
None)
def test_start_job_timeout(logger, callback):
mock_consumer.__iter__ = lambda x: iter([rec21])
worker = Worker(
hosts='localhost',
topic='foo',
timeout=1000,
callback=callback,
)
worker.start()
logger.info.assert_has_calls([
mock.call('Starting Worker(topic=foo) ...'),
mock.call('Processing {} ...'.format(rec21_repr)),
mock.call('Running Job 300: tests.utils.timeout_func(2, 3, 4) ...'),
mock.call('Executing callback ...')
])
logger.error.assert_called_once_with(
'Job 300 timed out after 100 seconds.')
callback.assert_called_once_with('timeout', timeout_job, None, None, None)
def test_start_job_callback_fail(logger, callback):
mock_consumer.__iter__ = lambda x: iter([rec11])
expected_error = KeyError('foo')
callback.side_effect = expected_error
worker = Worker(
hosts='localhost',
topic='foo',
callback=callback,
)
worker.start()
logger.info.assert_has_calls([
mock.call('Starting Worker(topic=foo) ...'),
mock.call('Processing {} ...'.format(rec11_repr)),
mock.call('Running Job 100: tests.utils.success_func(1, 2, c=3) ...'),
mock.call('Job 100 returned: (1, 2, 3)'),
mock.call('Executing callback ...')
])
logger.exception.assert_called_once_with(
'Callback failed: {}'.format(expected_error))
def test_start_job_failure(logger, callback):
mock_consumer.__iter__ = lambda x: iter([rec12])
worker = Worker(
hosts='localhost',
topic='foo',
timeout=1000,
callback=callback,
)
worker.start()
logger.info.assert_has_calls([
mock.call('Starting Worker(topic=foo) ...'),
mock.call('Processing {} ...'.format(rec12_repr)),
mock.call('Running Job 200: tests.utils.failure_func(1, 2, 3) ...'),
mock.call('Executing callback ...')
])
logger.exception.assert_called_with('Job 200 failed: failed!')
assert len(callback.call_args_list) == 1
callback_args = callback.call_args_list[0][0]
assert callback_args[0] == 'failure'
assert callback_args[1] == failure_job
assert callback_args[2] is None
assert isinstance(callback_args[3], ValueError)
assert isinstance(callback_args[4], str)
import logging
from kafka import KafkaConsumer
from kq import Worker
# Set up logging.
formatter = logging.Formatter('[%(levelname)s] %(message)s')
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger = logging.getLogger('kq.worker')
logger.setLevel(logging.DEBUG)
logger.addHandler(stream_handler)
# Set up a Kafka consumer.
consumer = KafkaConsumer(bootstrap_servers='34.73.142.27:9092',
group_id='group',
auto_offset_reset='latest')
# Set up a worker.
worker = Worker(topic='topic_queue', consumer=consumer)
worker.start()