def test__shoveller(self, easy_job_mock, produce_to_queue_mock):
job_mock = Mock()
job_mock.tag = "unknown"
easy_job_mock.create_from_dict.return_value = job_mock
body = json.dumps({"body": "work body"})
message = Mock()
api = "http://test.api.com/test_dest"
api_request_headers = {"title": "Yippi"}
job = EasyJob.create(api, constants.API_REMOTE, api_request_headers=api_request_headers)
headers = {}
headers.update(job.to_dict())
message.headers = headers
self.retry_consumer._shoveller(body, message)
produce_to_queue_mock.assert_called_with(constants.BUFFER_QUEUE, body, job_mock)
def test__push_msg_to_dlq(self, easy_job_mock, produce_to_queue):
job_mock = Mock()
job_mock.tag = "unknown"
job_mock.no_of_retries = 1
easy_job_mock.create_from_dict.return_value = job_mock
body = {"body": "work body"}
message = Mock()
api_request_headers = {"title": "Yippi"}
job = EasyJob.create("test_api",
constants.API_REMOTE,
api_request_headers=api_request_headers)
headers = {}
headers.update(job.to_dict())
message.headers = headers
self.work_queue_con._push_msg_to_dlq(body, message, job)
produce_to_queue.assert_called_with(constants.DEAD_LETTER_QUEUE, body,
job)
def test_process_message(self, easy_job_mock, produce_to_queue_mock):
# mock the job to be created in the process_message call
job_mock = Mock()
job_mock.tag = "unknown"
job_mock.no_of_retries = 1
easy_job_mock.create_from_dict.return_value = job_mock
body = json.dumps({"body": "work body"})
message = Mock()
api = "http://test.api.com/test_dest"
api_request_headers = {"title": "Yippi"}
job = EasyJob.create(api, constants.API_REMOTE, api_request_headers=api_request_headers)
headers = {}
headers.update(job.to_dict())
message.headers = headers
# when no of retires is less than max then add back to work queue
self.retry_consumer.process_message(body, message)
produce_to_queue_mock.assert_called_with(constants.WORK_QUEUE, body, job_mock)
message.ack.assert_called()
message.reset_mock()
# test exception
produce_to_queue_mock.side_effect = Exception()
self.retry_consumer.process_message(body, message)
message.assert_not_called()
message.reset_mock()
# when no of retries is more than max retries then add to dead letter queue
produce_to_queue_mock.side_effect = None
job_mock.no_of_retries = constants.DEFAULT_MAX_JOB_RETRIES + 1
self.retry_consumer.process_message(body, message)
produce_to_queue_mock.assert_called_with(constants.DEAD_LETTER_QUEUE, body, job_mock)
message.ack.assert_called()
# test exception
message.reset_mock()
produce_to_queue_mock.side_effect = Exception()
self.retry_consumer.process_message(body, message)
message.assert_not_called()
def enqueue_job(self,
api,
type,
tag=None,
remote_call_type=None,
data=None,
api_request_headers=None,
content_type=None,
notification_handler=None):
"""
Enqueue a job to be processed.
:param api: The api to be called when job is run
:param type: The type of job (Remote/Local) when local then a python call is made and in remote an REST call is made.
:param tag: a tag for the job to be run
:param remote_call_type: is the call POST/GET/PUT
:param data: a data payload to be passed along in the job
:param api_request_headers: request headers to be passed along in a remote call
:param content_type: content type to be used in remote call
:param notification_handler: the api to be called when a job goes into dlq (type same as api)
:return: A unique job id assigned to the job.
"""
self.validate_init()
# create the job
job = EasyJob.create(api,
type,
tag=tag,
remote_call_type=remote_call_type,
data=data,
api_request_headers=api_request_headers,
content_type=content_type,
notification_handler=notification_handler)
# enqueue
enqueue(self._producer, constants.WORK_QUEUE, job, data)
return job.id
def test_process_message(self, remote_call_type_mock, push_dlq_mock,
push_retry_mock):
post = Mock()
response = Mock()
response.status_code = 200
post.return_value = response
remote_call_type_mock.get.return_value = post
# Test remote job flow
body = json.dumps({"body": "work body"})
message = Mock()
api = "http://test.api.com/test_dest"
api_request_headers = {"title": "Yippi"}
job = EasyJob.create(api,
constants.API_REMOTE,
api_request_headers=api_request_headers)
headers = {}
headers.update(job.to_dict())
message.headers = headers
self.work_queue_con.process_message(body, message)
data_body = {'tag': 'unknown', 'data': body, 'job_id': job.id}
# when return in 200
post.assert_called_with(api,
data=data_body,
timeout=constants.DEFAULT_ASYNC_TIMEOUT,
headers=api_request_headers)
# when the status code is 410 (in the error list to be reported
# then the job will be added to be dlq
response.status_code = 410
response.text = "big error"
self.work_queue_con.process_message(body, message)
push_dlq_mock.assert_called()
# when the status code is 5XX then add to the error queue
response.status_code = 520
response.text = "big error"
self.work_queue_con.process_message(body, message)
push_retry_mock.assert_called()
# test local flow
sys.path += os.path.dirname(test_class.__file__)
# test with module function
api = test_class.dummy_function_external
job = EasyJob.create(api, constants.API_LOCAL)
headers = {}
headers.update(job.to_dict())
message.headers = headers
test_class.TestClass.module_function_called = False
self.work_queue_con.process_message(body, message)
self.assertEqual(test_class.TestClass.module_function_called, True)
# test with string function
api = "tests.test_class.dummy_function_external"
job = EasyJob.create(api, constants.API_LOCAL)
headers = {}
headers.update(job.to_dict())
message.headers = headers
test_class.TestClass.module_function_called = False
self.work_queue_con.process_message(body, message)
self.assertEqual(test_class.TestClass.module_function_called, True)
# test with instance function
test_cls = test_class.TestClass()
api = test_cls.dummy_function_in_class
job = EasyJob.create(api, constants.API_LOCAL)
headers = {}
headers.update(job.to_dict())
message.headers = headers
test_class.TestClass.class_function_called = False
self.work_queue_con.process_message(body, message)
self.assertEqual(test_class.TestClass.class_function_called, True)
# test with instance class
tst_class = test_class.TestClass()
job = EasyJob.create(tst_class, constants.API_LOCAL)
headers = {}
headers.update(job.to_dict())
message.headers = headers
test_class.TestClass.class_instance_called = False
self.work_queue_con.process_message(body, message)
self.assertEqual(test_class.TestClass.class_instance_called, True)