def test_task_complete(self):
task = ExternalTask({"id": "1", "topicName": "my_topic"})
output_vars = {"var1": 1, "var2": "value", "var3": True}
expected_task_result = TaskResult.success(task, output_vars)
external_task_client = ExternalTaskClient(worker_id=1)
responses.add(responses.POST, external_task_client.get_task_complete_url(task.get_task_id()),
status=HTTPStatus.NO_CONTENT)
executor = ExternalTaskExecutor(worker_id=1, external_task_client=external_task_client)
actual_task_result = executor.execute_task(task, self.task_success_action)
self.assertEqual(str(expected_task_result), str(actual_task_result))
def test_task_failure(self):
task = ExternalTask({"id": "1", "topicName": "my_topic"})
expected_task_result = TaskResult.failure(task,
error_message="unknown task failure", error_details="unknown error",
retries=3, retry_timeout=30000)
external_task_client = ExternalTaskClient(worker_id=1)
responses.add(responses.POST, external_task_client.get_task_failure_url(task.get_task_id()),
status=HTTPStatus.NO_CONTENT)
executor = ExternalTaskExecutor(worker_id=1, external_task_client=external_task_client)
actual_task_result = executor.execute_task(task, self.task_failure_action)
self.assertEqual(str(expected_task_result), str(actual_task_result))
def test_task_result_not_complete_failure_bpmnerror_raises_exception(self):
task = ExternalTask({"id": "1", "topicName": "my_topic"})
external_task_client = ExternalTaskClient(worker_id=1)
executor = ExternalTaskExecutor(
worker_id=1, external_task_client=external_task_client)
with self.assertRaises(Exception) as exception_ctx:
executor.execute_task(
task, self.task_result_not_complete_failure_bpmnerror)
self.assertEqual(
"task result for task_id=1 must be either complete/failure/BPMNError",
str(exception_ctx.exception))
def test_task_bpmn_error(self):
task = ExternalTask({"id": "1", "topicName": "my_topic"})
expected_task_result = TaskResult.bpmn_error(
task, error_code="bpmn_err_code_1", error_message="bpmn error")
external_task_client = ExternalTaskClient(worker_id=1)
responses.add(responses.POST,
external_task_client.get_task_bpmn_error_url(
task.get_task_id()),
status=HTTPStatus.NO_CONTENT)
executor = ExternalTaskExecutor(
worker_id=1, external_task_client=external_task_client)
actual_task_result = executor.execute_task(task,
self.task_bpmn_error_action)
self.assertEqual(str(expected_task_result), str(actual_task_result))
def test_execute_task_raises_exception_if_engine_returns_http_status_other_than_no_content_204(
self):
client = ExternalTaskClient(worker_id=1)
task = ExternalTask({"id": "1", "topicName": "my_topic"})
executor = ExternalTaskExecutor(worker_id=1,
external_task_client=client)
TaskResultStatusInput = collections.namedtuple(
'TaskResultStatusInput', [
'task_status', 'task_action', 'task_status_url',
'http_status_code', 'expected_error_message'
])
task_result_tests = [
TaskResultStatusInput(
"complete", self.task_success_action,
client.get_task_complete_url(task.get_task_id()),
HTTPStatus.OK,
'Not able to mark complete for task_id=1 for topic=my_topic, worker_id=1'
),
TaskResultStatusInput(
"failure", self.task_failure_action,
client.get_task_failure_url(task.get_task_id()),
HTTPStatus.CREATED,
'Not able to mark failure for task_id=1 for topic=my_topic, worker_id=1'
),
TaskResultStatusInput(
"bpmn_error", self.task_bpmn_error_action,
client.get_task_bpmn_error_url(task.get_task_id()),
HTTPStatus.ACCEPTED,
'Not able to mark BPMN Error for task_id=1 for topic=my_topic, worker_id=1'
)
]
for task_result_test in task_result_tests:
with self.subTest(task_result_test.task_status):
responses.add(responses.POST,
task_result_test.task_status_url,
status=task_result_test.http_status_code)
with self.assertRaises(Exception) as exception_ctx:
executor.execute_task(task, task_result_test.task_action)
self.assertEqual(task_result_test.expected_error_message,
str(exception_ctx.exception))
def test_execute_task_raises_exception_raised_when_updating_status_in_engine(
self):
client = ExternalTaskClient(worker_id=1)
task = ExternalTask({"id": "1", "topicName": "my_topic"})
executor = ExternalTaskExecutor(worker_id=1,
external_task_client=client)
TaskResultStatusInput = collections.namedtuple(
'TaskResultStatusInput',
['task_status', 'task_action', 'task_status_url', 'error_message'])
task_result_tests = [
TaskResultStatusInput(
"complete", self.task_success_action,
client.get_task_complete_url(task.get_task_id()),
"cannot update task status to complete"),
TaskResultStatusInput(
"failure", self.task_failure_action,
client.get_task_failure_url(task.get_task_id()),
"cannot update task status to failure"),
TaskResultStatusInput(
"bpmn_error", self.task_bpmn_error_action,
client.get_task_bpmn_error_url(task.get_task_id()),
"cannot update task status to BPMN err")
]
for task_result_test in task_result_tests:
with self.subTest(task_result_test.task_status):
responses.add(responses.POST,
task_result_test.task_status_url,
body=Exception(task_result_test.error_message))
with self.assertRaises(Exception) as exception_ctx:
executor.execute_task(task, task_result_test.task_action)
self.assertEqual(task_result_test.error_message,
str(exception_ctx.exception))
def test_strip_long_variables(self):
variables = {
"var0": "string",
"var1": {"value": "string"},
"var2": {"value": 1},
"var3": {"value": "{\"key\": \"value\"}", "type": "Json"},
"var4": {"value": base64.encodebytes(b"string"), "type": "Bytes"},
"var5": {"value": "some file content", "type": "File"},
}
cleaned = ExternalTaskExecutor._strip_long_variables(None, variables)
self.assertEqual({
"var0": "string",
"var1": {"value": "string"},
"var2": {"value": 1},
"var3": {"value": "{\"key\": \"value\"}", "type": "Json"},
"var4": {"value": "...", "type": "Bytes"},
"var5": {"value": "...", "type": "File"},
}, cleaned)
class ExternalTaskWorker:
DEFAULT_SLEEP_SECONDS = 300
def __init__(self, worker_id, base_url=ENGINE_LOCAL_BASE_URL, config=None):
config = config if config is not None else {
} # To avoid to have a mutable default for a parameter
self.worker_id = worker_id
self.client = ExternalTaskClient(self.worker_id, base_url, config)
self.executor = ExternalTaskExecutor(self.worker_id, self.client)
self.config = config
self._log_with_context(
f"Created new External Task Worker with config: {self.config}")
def subscribe(self, topic_names, action, process_variables=None):
while True:
self._fetch_and_execute_safe(topic_names, action,
process_variables)
self._log_with_context(
"Stopping worker") # Fixme: This code seems to be unreachable?
def _fetch_and_execute_safe(self,
topic_names,
action,
process_variables=None):
try:
self.fetch_and_execute(topic_names, action, process_variables)
except NoExternalTaskFound:
self._log_with_context(
f"no External Task found for Topics: {topic_names}, "
f"Process variables: {process_variables}",
topic=topic_names)
except BaseException as e:
sleep_seconds = self._get_sleep_seconds()
self._log_with_context(
f'error fetching and executing tasks: {get_exception_detail(e)} '
f'for topic(s)={topic_names} with Process variables: {process_variables}. '
f'retrying after {sleep_seconds} seconds',
exc_info=True)
time.sleep(sleep_seconds)
def fetch_and_execute(self, topic_names, action, process_variables=None):
self._log_with_context(
f"Fetching and Executing external tasks for Topics: {topic_names} "
f"with Process variables: {process_variables}")
resp_json = self._fetch_and_lock(topic_names, process_variables)
tasks = self._parse_response(resp_json, topic_names, process_variables)
if len(tasks) == 0:
raise NoExternalTaskFound(
f"no External Task found for Topics: {topic_names}, "
f"Process variables: {process_variables}")
self._execute_tasks(tasks, action)
def _fetch_and_lock(self, topic_names, process_variables=None):
self._log_with_context(
f"Fetching and Locking external tasks for Topics: {topic_names} "
f"with Process variables: {process_variables}")
return self.client.fetch_and_lock(topic_names, process_variables)
def _parse_response(self, resp_json, topic_names, process_variables):
tasks = []
if resp_json:
for context in resp_json:
task = ExternalTask(context)
tasks.append(task)
tasks_count = len(tasks)
self._log_with_context(
f"{tasks_count} External task(s) found for "
f"Topics: {topic_names}, Process variables: {process_variables}")
return tasks
def _execute_tasks(self, tasks, action):
for task in tasks:
self._execute_task(task, action)
def _execute_task(self, task, action):
try:
self.executor.execute_task(task, action)
except Exception as e:
self._log_with_context(
f'error when executing task: {get_exception_detail(e)}',
topic=task.get_topic_name(),
task_id=task.get_task_id(),
log_level='error',
exc_info=True)
raise e
def _log_with_context(self,
msg,
topic=None,
task_id=None,
log_level='info',
**kwargs):
context = {
"WORKER_ID": str(self.worker_id),
"TOPIC": topic,
"TASK_ID": task_id
}
log_with_context(msg, context=context, log_level=log_level, **kwargs)
def _get_sleep_seconds(self):
return self.config.get("sleepSeconds", self.DEFAULT_SLEEP_SECONDS)
def __init__(self, worker_id, base_url=ENGINE_LOCAL_BASE_URL, config=frozendict({})):
self.worker_id = worker_id
self.client = ExternalTaskClient(self.worker_id, base_url, config)
self.executor = ExternalTaskExecutor(self.worker_id, self.client)
self.config = config
self._log_with_context(f"Created new External Task Worker with config: {self.config}")