def test_execute_task_raises_exception_raised_when_updating_status_in_engine(self):
client = ExternalTaskClient(worker_id=1)
task = ExternalTask({"id": "1", "topicName": "my_topic"})
executor = ExternalTaskExecutor(worker_id=1, external_task_client=client)
TaskResultStatusInput = collections.namedtuple('TaskResultStatusInput',
['task_status', 'task_action', 'task_status_url',
'error_message'])
task_result_tests = [
TaskResultStatusInput("complete", self.task_success_action,
client.get_task_complete_url(task.get_task_id()),
"cannot update task status to complete"),
TaskResultStatusInput("failure", self.task_failure_action,
client.get_task_failure_url(task.get_task_id()),
"cannot update task status to failure"),
TaskResultStatusInput("bpmn_error", self.task_bpmn_error_action,
client.get_task_bpmn_error_url(task.get_task_id()),
"cannot update task status to BPMN err")
]
for task_result_test in task_result_tests:
with self.subTest(task_result_test.task_status):
responses.add(responses.POST, task_result_test.task_status_url,
body=Exception(task_result_test.error_message))
with self.assertRaises(Exception) as exception_ctx:
executor.execute_task(task, task_result_test.task_action)
self.assertEqual(task_result_test.error_message, str(exception_ctx.exception))
def test_execute_task_raises_exception_if_engine_returns_http_status_other_than_no_content_204(self):
client = ExternalTaskClient(worker_id=1)
task = ExternalTask({"id": "1", "topicName": "my_topic"})
executor = ExternalTaskExecutor(worker_id=1, external_task_client=client)
TaskResultStatusInput = collections.namedtuple('TaskResultStatusInput',
['task_status', 'task_action', 'task_status_url',
'http_status_code', 'expected_error_message'])
task_result_tests = [
TaskResultStatusInput("complete", self.task_success_action,
client.get_task_complete_url(task.get_task_id()), HTTPStatus.OK,
'Not able to mark complete for task_id=1 for topic=my_topic, worker_id=1'),
TaskResultStatusInput("failure", self.task_failure_action,
client.get_task_failure_url(task.get_task_id()), HTTPStatus.CREATED,
'Not able to mark failure for task_id=1 for topic=my_topic, worker_id=1'),
TaskResultStatusInput("bpmn_error", self.task_bpmn_error_action,
client.get_task_bpmn_error_url(task.get_task_id()), HTTPStatus.ACCEPTED,
'Not able to mark BPMN Error for task_id=1 for topic=my_topic, worker_id=1')
]
for task_result_test in task_result_tests:
with self.subTest(task_result_test.task_status):
responses.add(responses.POST, task_result_test.task_status_url,
status=task_result_test.http_status_code)
with self.assertRaises(Exception) as exception_ctx:
executor.execute_task(task, task_result_test.task_action)
self.assertEqual(task_result_test.expected_error_message, str(exception_ctx.exception))
def __init__(self, worker_id, base_url=ENGINE_LOCAL_BASE_URL, config=None):
config = config if config is not None else {
} # To avoid to have a mutable default for a parameter
self.worker_id = worker_id
self.client = ExternalTaskClient(self.worker_id, base_url, config)
self.executor = ExternalTaskExecutor(self.worker_id, self.client)
self.config = config
self._log_with_context(
f"Created new External Task Worker with config: {self.config}")
def test_task_bpmn_error(self):
task = ExternalTask({"id": "1", "topicName": "my_topic"})
expected_task_result = TaskResult.bpmn_error(task, error_code="bpmn_err_code_1", error_message="bpmn error")
external_task_client = ExternalTaskClient(worker_id=1)
responses.add(responses.POST, external_task_client.get_task_bpmn_error_url(task.get_task_id()),
status=HTTPStatus.NO_CONTENT)
executor = ExternalTaskExecutor(worker_id=1, external_task_client=external_task_client)
actual_task_result = executor.execute_task(task, self.task_bpmn_error_action)
self.assertEqual(str(expected_task_result), str(actual_task_result))
def test_task_complete(self):
task = ExternalTask({"id": "1", "topicName": "my_topic"})
output_vars = {"var1": 1, "var2": "value", "var3": True}
expected_task_result = TaskResult.success(task, output_vars)
external_task_client = ExternalTaskClient(worker_id=1)
responses.add(responses.POST, external_task_client.get_task_complete_url(task.get_task_id()),
status=HTTPStatus.NO_CONTENT)
executor = ExternalTaskExecutor(worker_id=1, external_task_client=external_task_client)
actual_task_result = executor.execute_task(task, self.task_success_action)
self.assertEqual(str(expected_task_result), str(actual_task_result))
def test_task_failure(self):
task = ExternalTask({"id": "1", "topicName": "my_topic"})
expected_task_result = TaskResult.failure(task,
error_message="unknown task failure", error_details="unknown error",
retries=3, retry_timeout=30000)
external_task_client = ExternalTaskClient(worker_id=1)
responses.add(responses.POST, external_task_client.get_task_failure_url(task.get_task_id()),
status=HTTPStatus.NO_CONTENT)
executor = ExternalTaskExecutor(worker_id=1, external_task_client=external_task_client)
actual_task_result = executor.execute_task(task, self.task_failure_action)
self.assertEqual(str(expected_task_result), str(actual_task_result))
def test_fetch_and_execute_safe_raises_exception_sleep_is_called(
self, mock_time_sleep):
external_task_client = ExternalTaskClient(worker_id=0)
responses.add(responses.POST,
external_task_client.get_fetch_and_lock_url(),
status=HTTPStatus.INTERNAL_SERVER_ERROR)
sleep_seconds = 100
worker = ExternalTaskWorker(worker_id=0,
config={"sleepSeconds": sleep_seconds})
mock_action = mock.Mock()
worker._fetch_and_execute_safe("my_topic", mock_action)
self.assertEqual(0, mock_action.call_count)
self.assertEqual(1, mock_time_sleep.call_count)
mock_time_sleep.assert_called_with(sleep_seconds)
def test_fetch_and_execute_raises_exception_if_no_tasks_found(self):
external_task_client = ExternalTaskClient(worker_id=0)
resp_payload = []
responses.add(responses.POST,
external_task_client.get_fetch_and_lock_url(),
status=HTTPStatus.OK,
json=resp_payload)
worker = ExternalTaskWorker(worker_id=0)
mock_action = mock.Mock()
process_variables = {"var1": "value1", "var2": "value2"}
with self.assertRaises(Exception) as context:
worker.fetch_and_execute("my_topic", mock_action,
process_variables)
self.assertEqual(
f"no External Task found for Topics: my_topic, Process variables: {process_variables}",
str(context.exception))
def test_task_result_not_complete_failure_bpmnerror_raises_exception(self):
task = ExternalTask({"id": "1", "topicName": "my_topic"})
external_task_client = ExternalTaskClient(worker_id=1)
executor = ExternalTaskExecutor(worker_id=1, external_task_client=external_task_client)
with self.assertRaises(Exception) as exception_ctx:
executor.execute_task(task, self.task_result_not_complete_failure_bpmnerror)
self.assertEqual("task result for task_id=1 must be either complete/failure/BPMNError",
str(exception_ctx.exception))
def test_fetch_and_execute_raises_exception_if_task_action_raises_exception(
self):
external_task_client = ExternalTaskClient(worker_id=0)
resp_payload = [{
"activityId": "anActivityId",
"activityInstanceId": "anActivityInstanceId",
"errorMessage": "anErrorMessage",
"errorDetails": "anErrorDetails",
"executionId": "anExecutionId",
"id": "anExternalTaskId",
"lockExpirationTime": "2015-10-06T16:34:42",
"processDefinitionId": "aProcessDefinitionId",
"processDefinitionKey": "aProcessDefinitionKey",
"processInstanceId": "aProcessInstanceId",
"tenantId": None,
"retries": 3,
"workerId": "aWorkerId",
"priority": 4,
"topicName": "createOrder",
"variables": {
"orderId": {
"type": "String",
"value": "1234",
"valueInfo": {}
}
}
}]
responses.add(responses.POST,
external_task_client.get_fetch_and_lock_url(),
status=HTTPStatus.OK,
json=resp_payload)
worker = ExternalTaskWorker(worker_id=0)
mock_action = mock.Mock()
mock_action.side_effect = Exception("error executing task action")
with self.assertRaises(Exception) as exception_ctx:
worker.fetch_and_execute("my_topic", mock_action)
self.assertEqual("error executing task action",
str(exception_ctx.exception))
class ExternalTaskWorker:
DEFAULT_SLEEP_SECONDS = 300
def __init__(self, worker_id, base_url=ENGINE_LOCAL_BASE_URL, config=None):
config = config if config is not None else {
} # To avoid to have a mutable default for a parameter
self.worker_id = worker_id
self.client = ExternalTaskClient(self.worker_id, base_url, config)
self.executor = ExternalTaskExecutor(self.worker_id, self.client)
self.config = config
self._log_with_context(
f"Created new External Task Worker with config: {self.config}")
def subscribe(self, topic_names, action, process_variables=None):
while True:
self._fetch_and_execute_safe(topic_names, action,
process_variables)
self._log_with_context(
"Stopping worker") # Fixme: This code seems to be unreachable?
def _fetch_and_execute_safe(self,
topic_names,
action,
process_variables=None):
try:
self.fetch_and_execute(topic_names, action, process_variables)
except NoExternalTaskFound:
self._log_with_context(
f"no External Task found for Topics: {topic_names}, "
f"Process variables: {process_variables}",
topic=topic_names)
except BaseException as e:
sleep_seconds = self._get_sleep_seconds()
self._log_with_context(
f'error fetching and executing tasks: {get_exception_detail(e)} '
f'for topic(s)={topic_names} with Process variables: {process_variables}. '
f'retrying after {sleep_seconds} seconds',
exc_info=True)
time.sleep(sleep_seconds)
def fetch_and_execute(self, topic_names, action, process_variables=None):
self._log_with_context(
f"Fetching and Executing external tasks for Topics: {topic_names} "
f"with Process variables: {process_variables}")
resp_json = self._fetch_and_lock(topic_names, process_variables)
tasks = self._parse_response(resp_json, topic_names, process_variables)
if len(tasks) == 0:
raise NoExternalTaskFound(
f"no External Task found for Topics: {topic_names}, "
f"Process variables: {process_variables}")
self._execute_tasks(tasks, action)
def _fetch_and_lock(self, topic_names, process_variables=None):
self._log_with_context(
f"Fetching and Locking external tasks for Topics: {topic_names} "
f"with Process variables: {process_variables}")
return self.client.fetch_and_lock(topic_names, process_variables)
def _parse_response(self, resp_json, topic_names, process_variables):
tasks = []
if resp_json:
for context in resp_json:
task = ExternalTask(context)
tasks.append(task)
tasks_count = len(tasks)
self._log_with_context(
f"{tasks_count} External task(s) found for "
f"Topics: {topic_names}, Process variables: {process_variables}")
return tasks
def _execute_tasks(self, tasks, action):
for task in tasks:
self._execute_task(task, action)
def _execute_task(self, task, action):
try:
self.executor.execute_task(task, action)
except Exception as e:
self._log_with_context(
f'error when executing task: {get_exception_detail(e)}',
topic=task.get_topic_name(),
task_id=task.get_task_id(),
log_level='error',
exc_info=True)
raise e
def _log_with_context(self,
msg,
topic=None,
task_id=None,
log_level='info',
**kwargs):
context = {
"WORKER_ID": str(self.worker_id),
"TOPIC": topic,
"TASK_ID": task_id
}
log_with_context(msg, context=context, log_level=log_level, **kwargs)
def _get_sleep_seconds(self):
return self.config.get("sleepSeconds", self.DEFAULT_SLEEP_SECONDS)
def __init__(self, worker_id, base_url=ENGINE_LOCAL_BASE_URL, config=frozendict({})):
self.worker_id = worker_id
self.client = ExternalTaskClient(self.worker_id, base_url, config)
self.executor = ExternalTaskExecutor(self.worker_id, self.client)
self.config = config
self._log_with_context(f"Created new External Task Worker with config: {self.config}")
def test_fetch_and_execute_calls_task_action_for_each_task_fetched(
self, mock_client):
external_task_client = ExternalTaskClient(worker_id=0)
resp_payload = [{
"activityId": "anActivityId",
"activityInstanceId": "anActivityInstanceId",
"errorMessage": "anErrorMessage",
"errorDetails": "anErrorDetails",
"executionId": "anExecutionId",
"id": "anExternalTaskId",
"lockExpirationTime": "2015-10-06T16:34:42",
"processDefinitionId": "aProcessDefinitionId",
"processDefinitionKey": "aProcessDefinitionKey",
"processInstanceId": "aProcessInstanceId",
"tenantId": None,
"retries": 3,
"workerId": "aWorkerId",
"priority": 4,
"topicName": "createOrder",
"variables": {
"orderId": {
"type": "String",
"value": "1234",
"valueInfo": {}
}
}
}, {
"activityId": "anActivityId",
"activityInstanceId": "anActivityInstanceId",
"errorMessage": "anErrorMessage",
"errorDetails": "anotherErrorDetails",
"executionId": "anExecutionId",
"id": "anExternalTaskId",
"lockExpirationTime": "2015-10-06T16:34:42",
"processDefinitionId": "aProcessDefinitionId",
"processDefinitionKey": "aProcessDefinitionKey",
"processInstanceId": "aProcessInstanceId",
"tenantId": None,
"retries": 3,
"workerId": "aWorkerId",
"priority": 0,
"topicName": "createOrder",
"variables": {
"orderId": {
"type": "String",
"value": "3456",
"valueInfo": {}
}
}
}]
responses.add(responses.POST,
external_task_client.get_fetch_and_lock_url(),
status=HTTPStatus.OK,
json=resp_payload)
worker = ExternalTaskWorker(worker_id=0)
mock_action = mock.Mock()
task = ExternalTask({
"id": "anExternalTaskId",
"workerId": "aWorkerId",
"topicName": "createOrder"
})
mock_action.return_value = TaskResult.success(task=task,
global_variables={})
worker.fetch_and_execute("my_topic", mock_action)
self.assertEqual(2, mock_action.call_count)
def test_creation_with_no_debug_config(self):
client = ExternalTaskClient(1, ENGINE_LOCAL_BASE_URL, {})
self.assertFalse(client.is_debug)
self.assertFalse(client.config.get("isDebug"))