def process_message(self, body, message):
"""
gets called back once a message arrives in the work queue
1. Sends email notifiactions once msg goes to dead letter queue
:param body: message payload
:param message: queued message with headers and other metadata
"""
logger = logging.getLogger(self.__class__.__name__)
job = EasyJob.create_from_dict(message.headers)
try:
response = job.notify_error(body, self.get_config().async_timeout)
if response.status_code != 200:
logger.error("Failed to notify with status code: {} with message: {}".format(response.status_code,
response.message))
self.log_to_file(response, job, body)
except Exception as e:
traceback.print_exc()
err_msg = "Notification Failed for job {}".format(body)
logger.error(err_msg)
message.ack()
def test__shoveller(self, easy_job_mock, produce_to_queue_mock):
job_mock = Mock()
job_mock.tag = "unknown"
easy_job_mock.create_from_dict.return_value = job_mock
body = json.dumps({"body": "work body"})
message = Mock()
api = "http://test.api.com/test_dest"
api_request_headers = {"title": "Yippi"}
job = EasyJob.create(api, constants.API_REMOTE, api_request_headers=api_request_headers)
headers = {}
headers.update(job.to_dict())
message.headers = headers
self.retry_consumer._shoveller(body, message)
produce_to_queue_mock.assert_called_with(constants.BUFFER_QUEUE, body, job_mock)
def __push_raw_msg_to_dlq(self, body, message, err_msg):
"""
pushes the raw message to dead letter queue for manual intervension and notification
:param body: body of the message
:param message: kombu amqp message object with headers and other metadata
:param error_mesg: what error caused this push to error queue
"""
logger = logging.getLogger(self.__class__.__name__)
try:
logger.info("Moving raw item to DLQ for notification and manual intervention")
job = EasyJob.create_dummy_clone_from_dict(message.headers)
job.add_error(EasyResponse(400, err_msg).__dict__)
self.produce_to_queue(constants.DEAD_LETTER_QUEUE, body, job)
except Exception as e:
traceback.print_exc()
logger.error("Error moving the raw-error to dead-letter-queue: {err}".format(err=str(e)))
def test__push_msg_to_dlq(self, easy_job_mock, produce_to_queue):
job_mock = Mock()
job_mock.tag = "unknown"
job_mock.no_of_retries = 1
easy_job_mock.create_from_dict.return_value = job_mock
body = {"body": "work body"}
message = Mock()
api_request_headers = {"title": "Yippi"}
job = EasyJob.create("test_api",
constants.API_REMOTE,
api_request_headers=api_request_headers)
headers = {}
headers.update(job.to_dict())
message.headers = headers
self.work_queue_con._push_msg_to_dlq(body, message, job)
produce_to_queue.assert_called_with(constants.DEAD_LETTER_QUEUE, body,
job)
def test_process_message(self, easy_job_mock, produce_to_queue_mock):
# mock the job to be created in the process_message call
job_mock = Mock()
job_mock.tag = "unknown"
job_mock.no_of_retries = 1
easy_job_mock.create_from_dict.return_value = job_mock
body = json.dumps({"body": "work body"})
message = Mock()
api = "http://test.api.com/test_dest"
api_request_headers = {"title": "Yippi"}
job = EasyJob.create(api, constants.API_REMOTE, api_request_headers=api_request_headers)
headers = {}
headers.update(job.to_dict())
message.headers = headers
# when no of retires is less than max then add back to work queue
self.retry_consumer.process_message(body, message)
produce_to_queue_mock.assert_called_with(constants.WORK_QUEUE, body, job_mock)
message.ack.assert_called()
message.reset_mock()
# test exception
produce_to_queue_mock.side_effect = Exception()
self.retry_consumer.process_message(body, message)
message.assert_not_called()
message.reset_mock()
# when no of retries is more than max retries then add to dead letter queue
produce_to_queue_mock.side_effect = None
job_mock.no_of_retries = constants.DEFAULT_MAX_JOB_RETRIES + 1
self.retry_consumer.process_message(body, message)
produce_to_queue_mock.assert_called_with(constants.DEAD_LETTER_QUEUE, body, job_mock)
message.ack.assert_called()
# test exception
message.reset_mock()
produce_to_queue_mock.side_effect = Exception()
self.retry_consumer.process_message(body, message)
message.assert_not_called()
def enqueue_job(self,
api,
type,
tag=None,
remote_call_type=None,
data=None,
api_request_headers=None,
content_type=None,
notification_handler=None):
"""
Enqueue a job to be processed.
:param api: The api to be called when job is run
:param type: The type of job (Remote/Local) when local then a python call is made and in remote an REST call is made.
:param tag: a tag for the job to be run
:param remote_call_type: is the call POST/GET/PUT
:param data: a data payload to be passed along in the job
:param api_request_headers: request headers to be passed along in a remote call
:param content_type: content type to be used in remote call
:param notification_handler: the api to be called when a job goes into dlq (type same as api)
:return: A unique job id assigned to the job.
"""
self.validate_init()
# create the job
job = EasyJob.create(api,
type,
tag=tag,
remote_call_type=remote_call_type,
data=data,
api_request_headers=api_request_headers,
content_type=content_type,
notification_handler=notification_handler)
# enqueue
enqueue(self._producer, constants.WORK_QUEUE, job, data)
return job.id
def test_process_message(self, remote_call_type_mock, push_dlq_mock,
push_retry_mock):
post = Mock()
response = Mock()
response.status_code = 200
post.return_value = response
remote_call_type_mock.get.return_value = post
# Test remote job flow
body = json.dumps({"body": "work body"})
message = Mock()
api = "http://test.api.com/test_dest"
api_request_headers = {"title": "Yippi"}
job = EasyJob.create(api,
constants.API_REMOTE,
api_request_headers=api_request_headers)
headers = {}
headers.update(job.to_dict())
message.headers = headers
self.work_queue_con.process_message(body, message)
data_body = {'tag': 'unknown', 'data': body, 'job_id': job.id}
# when return in 200
post.assert_called_with(api,
data=data_body,
timeout=constants.DEFAULT_ASYNC_TIMEOUT,
headers=api_request_headers)
# when the status code is 410 (in the error list to be reported
# then the job will be added to be dlq
response.status_code = 410
response.text = "big error"
self.work_queue_con.process_message(body, message)
push_dlq_mock.assert_called()
# when the status code is 5XX then add to the error queue
response.status_code = 520
response.text = "big error"
self.work_queue_con.process_message(body, message)
push_retry_mock.assert_called()
# test local flow
sys.path += os.path.dirname(test_class.__file__)
# test with module function
api = test_class.dummy_function_external
job = EasyJob.create(api, constants.API_LOCAL)
headers = {}
headers.update(job.to_dict())
message.headers = headers
test_class.TestClass.module_function_called = False
self.work_queue_con.process_message(body, message)
self.assertEqual(test_class.TestClass.module_function_called, True)
# test with string function
api = "tests.test_class.dummy_function_external"
job = EasyJob.create(api, constants.API_LOCAL)
headers = {}
headers.update(job.to_dict())
message.headers = headers
test_class.TestClass.module_function_called = False
self.work_queue_con.process_message(body, message)
self.assertEqual(test_class.TestClass.module_function_called, True)
# test with instance function
test_cls = test_class.TestClass()
api = test_cls.dummy_function_in_class
job = EasyJob.create(api, constants.API_LOCAL)
headers = {}
headers.update(job.to_dict())
message.headers = headers
test_class.TestClass.class_function_called = False
self.work_queue_con.process_message(body, message)
self.assertEqual(test_class.TestClass.class_function_called, True)
# test with instance class
tst_class = test_class.TestClass()
job = EasyJob.create(tst_class, constants.API_LOCAL)
headers = {}
headers.update(job.to_dict())
message.headers = headers
test_class.TestClass.class_instance_called = False
self.work_queue_con.process_message(body, message)
self.assertEqual(test_class.TestClass.class_instance_called, True)
def process_message(self, body, message):
"""
gets called back once a message arrives in the work queue
1. calls embedded api with the payload as its parameters when a message arrives
2. if the call is successful, acks the message
3. for remote call
a. in case the call fails with 4XX, just acks the message, no further action
b. in case the call fails with a 5XX,
- adds the error to error-log header
- if num-retries are more than max_retries,
- puts the message in dead-letter-queue
- else
- increases num-retries by 1
- puts the message in error-queue
4. for local call
a. in case the call fails with a exception then adds the call to a dead letter queue
:param body: message payload
:param message: queued message with headers and other metadata (contains a EasyJob object in headers)
"""
logger = logging.getLogger(self.__class__.__name__)
try:
job = EasyJob.create_from_dict(message.headers)
except easyjoblite.exception.UnableToCreateJob as e:
logger.error(e.message + " data: " + str(e.data))
message.ack()
self.__push_raw_msg_to_dlq(body=body,
message=message,
err_msg=e.message,
)
return
try:
api = job.api
logger.debug("recieved api: " + str(api))
response = job.execute(body, self.get_config().async_timeout)
message.ack()
if response.status_code >= 400:
# todo: we should have booking id here in the log message
logger.info("{status}: {resp}".format(status=response.status_code,
resp=response.message))
if response.status_code >= 500:
# we have a retry-able failure
self._push_message_to_error_queue(body=body, message=message, job=job)
else:
# push not retry-able error to dlq
self._push_msg_to_dlq(body=body,
message=message,
job=job
)
except (Exception, easyjoblite.exception.ApiTimeoutException) as e:
traceback.print_exc()
logger.error(str(e))
message.ack()
self._push_message_to_error_queue(body, message, job)