def _process_message(self, msg: Message) -> None:
"""De-serialize the message and execute the job.
:param msg: Kafka message.
:type msg: :doc:`kq.Message <message>`
"""
self._logger.info(
"Processing Message(topic={}, partition={}, offset={}) ...".format(
msg.topic, msg.partition, msg.offset))
try:
job = self._deserializer(msg.value)
job_repr = get_call_repr(job.func, *job.args, **job.kwargs)
except Exception as err:
self._logger.exception(f"Job was invalid: {err}")
self._execute_callback("invalid", msg, None, None, None, None)
else:
self._logger.info(f"Executing job {job.id}: {job_repr}")
timer: Optional[threading.Timer]
if job.timeout:
timer = threading.Timer(job.timeout, _thread.interrupt_main)
timer.start()
else:
timer = None
try:
res = job.func(*job.args, **job.kwargs)
except KeyboardInterrupt:
self._logger.error(
f"Job {job.id} timed out or was interrupted")
self._execute_callback("timeout", msg, job, None, None, None)
except Exception as err:
self._logger.exception(f"Job {job.id} raised an exception:")
tb = traceback.format_exc()
self._execute_callback("failure", msg, job, None, err, tb)
else:
self._logger.info(f"Job {job.id} returned: {res}")
self._execute_callback("success", msg, job, res, None, None)
finally:
if timer is not None:
timer.cancel()
def _process_message(self, msg):
"""De-serialize the message and execute the job.
:param msg: Kafka message.
:type msg: :doc:`kq.Message <message>`
"""
self._logger.info(
'Processing Message(topic={}, partition={}, offset={}) ...'.format(
msg.topic, msg.partition, msg.offset))
try:
job = self._deserializer(msg.value)
job_repr = get_call_repr(job.func, *job.args, **job.kwargs)
except Exception as err:
self._logger.exception('Job was invalid: {}'.format(err))
self._execute_callback('invalid', msg, None, None, None, None)
else:
self._logger.info('Executing job {}: {}'.format(job.id, job_repr))
if job.timeout:
timer = threading.Timer(job.timeout, _thread.interrupt_main)
timer.start()
else:
timer = None
try:
res = job.func(*job.args, **job.kwargs)
except KeyboardInterrupt:
self._logger.error(
'Job {} timed out or was interrupted'.format(job.id))
self._execute_callback('timeout', msg, job, None, None, None)
except Exception as err:
self._logger.exception('Job {} raised an exception:'.format(
job.id))
tb = traceback.format_exc()
self._execute_callback('failure', msg, job, None, err, tb)
else:
self._logger.info('Job {} returned: {}'.format(job.id, res))
self._execute_callback('success', msg, job, res, None, None)
finally:
if timer is not None:
timer.cancel()
def test_call_repr_complex_args(timeout_func):
expected = 'tests.conftest.timeout_function([1])'
assert expected == get_call_repr(timeout_func, [1])
expected = 'tests.conftest.timeout_function([1], [2])'
assert expected == get_call_repr(timeout_func, [1], [2])
expected = 'tests.conftest.timeout_function([1], b=[2])'
assert expected == get_call_repr(timeout_func, [1], b=[2])
expected = 'tests.conftest.timeout_function(a=[1])'
assert expected == get_call_repr(timeout_func, a=[1])
expected = 'tests.conftest.timeout_function(b=[1])'
assert expected == get_call_repr(timeout_func, b=[1])
expected = 'tests.conftest.timeout_function(a=[1], b=[1, 2])'
assert expected == get_call_repr(timeout_func, a=[1], b=[1, 2])
expected = 'tests.conftest.timeout_function(a=[1], b=[1, 2])'
assert expected == get_call_repr(timeout_func, b=[1, 2], a=[1])
def test_call_repr_callable_types(success_func, callable_cls):
expected = 'None()'
assert expected == get_call_repr(None)
expected = 'builtins.isinstance()'
assert expected == get_call_repr(isinstance)
expected = 'tests.conftest.success_function()'
assert expected == get_call_repr(success_func)
expected = 'tests.conftest.success_function()'
assert expected == get_call_repr(callable_cls.unbound_method)
expected = 'tests.conftest.Callable.static_method()'
assert expected == get_call_repr(callable_cls.static_method)
expected = 'tests.conftest.Callable.instance_method()'
assert expected == get_call_repr(callable_cls().instance_method)
expected = 'tests.conftest.Callable()'
assert expected == get_call_repr(callable_cls())
def test_call_repr_simple_args(failure_func):
expected = 'tests.conftest.failure_function(1)'
assert expected == get_call_repr(failure_func, 1)
expected = 'tests.conftest.failure_function(1, 2)'
assert expected == get_call_repr(failure_func, 1, 2)
expected = 'tests.conftest.failure_function(1, b=2)'
assert expected == get_call_repr(failure_func, 1, b=2)
expected = 'tests.conftest.failure_function(a=1)'
assert expected == get_call_repr(failure_func, a=1)
expected = 'tests.conftest.failure_function(b=1)'
assert expected == get_call_repr(failure_func, b=1)
expected = 'tests.conftest.failure_function(a=1, b=2)'
assert expected == get_call_repr(failure_func, a=1, b=2)
expected = 'tests.conftest.failure_function(a=1, b=2)'
assert expected == get_call_repr(failure_func, b=2, a=1)