async def _run(self):
async with aiohttp.ClientSession() as session:
self.worker = ExternalTaskWorker(
worker_id=4, base_url="http://localhost:8080/engine-rest", session=session
)
# dispatch the first subscription
self.loop.create_task(
self.worker.subscribe(topic_names="NumberCheckTask", action=number_check)
)
# and block the current task with the second subscription again
await self.worker.subscribe(topic_names="EchoTask", action=echo)
async def main():
configure_logging()
loop = asyncio.get_event_loop()
iovation_topic = loop.create_task(
ExternalTaskWorker(config=custom_config).subscribe(
["GET_IOVATION_DATA"], get_iovation_data))
sentilink_topic = loop.create_task(
ExternalTaskWorker(config=custom_config).subscribe(
["GET_SENTILINK_DATA"], get_sentilink_data))
await asyncio.gather(iovation_topic, sentilink_topic)
def test_fetch_and_execute_safe_raises_exception_sleep_is_called(
self, mock_time_sleep):
external_task_client = ExternalTaskClient(worker_id=0)
responses.add(responses.POST,
external_task_client.get_fetch_and_lock_url(),
status=HTTPStatus.INTERNAL_SERVER_ERROR)
sleep_seconds = 100
worker = ExternalTaskWorker(worker_id=0,
config={"sleepSeconds": sleep_seconds})
mock_action = mock.Mock()
worker._fetch_and_execute_safe("my_topic", mock_action)
self.assertEqual(0, mock_action.call_count)
self.assertEqual(1, mock_time_sleep.call_count)
mock_time_sleep.assert_called_with(sleep_seconds)
async def main():
async with aiohttp.ClientSession() as session:
# a new parameter! We can pass 'asyncResponseTimeout' to Camunda to configure how long a 'fetch and lock' connection
# is kept alive. If this value is rather large, clients do not need to reconnect that often but it will take longer for a
# worker to properly shut down. The default value is 30 seconds (300000)
worker = ExternalTaskWorker(
worker_id=4,
base_url="http://localhost:8080/engine-rest",
session=session,
config={"asyncResponseTimeout":
5000}, # wait 5 seconds before timeout
)
# Our worker will now subscribe to two topics now
# We will create a new task with `asyncio.create_task` and await only the second subscribe
asyncio.create_task(
worker.subscribe(topic_names="NumberCheckTask",
action=number_check))
await worker.subscribe(topic_names="EchoTask", action=echo)
def main():
configure_logging()
topics = ["PARALLEL_STEP_1", "PARALLEL_STEP_2", "COMBINE_STEP"]
executor = ThreadPoolExecutor(max_workers=len(topics))
for index, topic in enumerate(topics):
executor.submit(
ExternalTaskWorker(worker_id=index,
config=default_config).subscribe, topic,
handle_task, {"strVar": "hello"})
def test_fetch_and_execute_raises_exception_if_no_tasks_found(self):
external_task_client = ExternalTaskClient(worker_id=0)
resp_payload = []
responses.add(responses.POST,
external_task_client.get_fetch_and_lock_url(),
status=HTTPStatus.OK,
json=resp_payload)
worker = ExternalTaskWorker(worker_id=0)
mock_action = mock.Mock()
process_variables = {"var1": "value1", "var2": "value2"}
with self.assertRaises(Exception) as context:
worker.fetch_and_execute("my_topic", mock_action,
process_variables)
self.assertEqual(
f"no External Task found for Topics: my_topic, Process variables: {process_variables}",
str(context.exception))
def main():
configure_logging()
topics = [
("TASK_1", fail_task_handler),
("TASK_2", fail_task_handler),
]
executor = ThreadPoolExecutor(max_workers=len(topics))
for index, topic_handler in enumerate(topics):
topic = topic_handler[0]
handler_func = topic_handler[1]
executor.submit(ExternalTaskWorker(worker_id=index, config=default_config).subscribe, topic, handler_func)
def test_fetch_and_execute_raises_exception_if_task_action_raises_exception(
self):
external_task_client = ExternalTaskClient(worker_id=0)
resp_payload = [{
"activityId": "anActivityId",
"activityInstanceId": "anActivityInstanceId",
"errorMessage": "anErrorMessage",
"errorDetails": "anErrorDetails",
"executionId": "anExecutionId",
"id": "anExternalTaskId",
"lockExpirationTime": "2015-10-06T16:34:42",
"processDefinitionId": "aProcessDefinitionId",
"processDefinitionKey": "aProcessDefinitionKey",
"processInstanceId": "aProcessInstanceId",
"tenantId": None,
"retries": 3,
"workerId": "aWorkerId",
"priority": 4,
"topicName": "createOrder",
"variables": {
"orderId": {
"type": "String",
"value": "1234",
"valueInfo": {}
}
}
}]
responses.add(responses.POST,
external_task_client.get_fetch_and_lock_url(),
status=HTTPStatus.OK,
json=resp_payload)
worker = ExternalTaskWorker(worker_id=0)
mock_action = mock.Mock()
mock_action.side_effect = Exception("error executing task action")
with self.assertRaises(Exception) as exception_ctx:
worker.fetch_and_execute("my_topic", mock_action)
self.assertEqual("error executing task action",
str(exception_ctx.exception))
def main():
configure_logging()
topics = [("VALIDATE_IMAGE", validate_image),
# ("APPROVE_IMAGE", generic_task_handler),
# ("REJECT_IMAGE", generic_task_handler),
# ("ENHANCE_IMAGE_QUALITY", generic_task_handler),
]
executor = ThreadPoolExecutor(max_workers=len(topics))
for index, topic_handler in enumerate(topics):
topic = topic_handler[0]
handler_func = topic_handler[1]
executor.submit(ExternalTaskWorker(worker_id=index, config=default_config).subscribe, topic, handler_func)
class Worker:
def __init__(self):
self.worker = None
self.loop = None
def start(self):
"""Run the worker and block forever"""
self.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self._run())
async def _run(self):
async with aiohttp.ClientSession() as session:
self.worker = ExternalTaskWorker(
worker_id=4, base_url="http://localhost:8080/engine-rest", session=session
)
# dispatch the first subscription
self.loop.create_task(
self.worker.subscribe(topic_names="NumberCheckTask", action=number_check)
)
# and block the current task with the second subscription again
await self.worker.subscribe(topic_names="EchoTask", action=echo)
def stop(self):
self.loop.run_until_complete(self.worker.cancel())
def main():
logger = get_logger()
logger.info("Workers started")
BASE_URL = "http://camunda_acmesky:8080/engine-rest"
""" Topics associated to the tasks
"""
TOPICS = register_user_interest_TASKS + last_minute_notifications_TASKS + daily_fligh_check_TASKS + buy_offer_TASKS
# Setup PostgreSQL
Base.metadata.create_all(create_sql_engine())
"""
Creation and execution of different threads, one per worker/topic
"""
executor = ThreadPoolExecutor(max_workers=len(TOPICS),
thread_name_prefix="ACMESky-Backend")
for index, topic_handler in enumerate(TOPICS):
topic = topic_handler[0]
handler_func = topic_handler[1]
executor.submit(
ExternalTaskWorker(worker_id=index,
base_url=BASE_URL,
config=default_config).subscribe, topic,
handler_func)
def main():
configure_logging()
topics = ["PARALLEL_STEP_1", "PARALLEL_STEP_2", "COMBINE_STEP"]
for index, topic in enumerate(topics):
ExternalTaskWorker(worker_id=index, config=default_config) \
.fetch_and_execute(topic_names=topic, action=handle_task, process_variables={"strVar": "hello"})
car_dict = json.loads(cars)
de = []
da = []
for key in car_dict:
if car_dict.get(key).get('country') == 'DE':
de.append(car_dict.get(key).get('price'))
if car_dict.get(key).get('country') == 'DK':
da.append(car_dict.get(key).get('price'))
duty = (sum(da) / len(da)) - (sum(de) / len(de))
# mark task either complete/failure/bpmnError based on outcome of your business logic
# if failure:
# # this marks task as failed in Camunda
# return task.failure(error_message="task failed", error_details="failed task details",
# max_retries=3, retry_timeout=5000)
# elif bpmn_error:
# return task.bpmn_error(error_code="BPMN_ERROR_CODE", error_message="BPMN Error occurred",
# variables={"var1": "value1", "success": False})
# pass any output variables you may want to send to Camunda as dictionary to complete()
return task.complete({"duty": duty})
if __name__ == '__main__':
print('worker started', flush=True)
ExternalTaskWorker(worker_id="1",
base_url="http://camunda:8080/engine-rest",
config=default_config).subscribe(
"dutycal", handle_task)
def test_fetch_and_execute_calls_task_action_for_each_task_fetched(
self, mock_client):
external_task_client = ExternalTaskClient(worker_id=0)
resp_payload = [{
"activityId": "anActivityId",
"activityInstanceId": "anActivityInstanceId",
"errorMessage": "anErrorMessage",
"errorDetails": "anErrorDetails",
"executionId": "anExecutionId",
"id": "anExternalTaskId",
"lockExpirationTime": "2015-10-06T16:34:42",
"processDefinitionId": "aProcessDefinitionId",
"processDefinitionKey": "aProcessDefinitionKey",
"processInstanceId": "aProcessInstanceId",
"tenantId": None,
"retries": 3,
"workerId": "aWorkerId",
"priority": 4,
"topicName": "createOrder",
"variables": {
"orderId": {
"type": "String",
"value": "1234",
"valueInfo": {}
}
}
}, {
"activityId": "anActivityId",
"activityInstanceId": "anActivityInstanceId",
"errorMessage": "anErrorMessage",
"errorDetails": "anotherErrorDetails",
"executionId": "anExecutionId",
"id": "anExternalTaskId",
"lockExpirationTime": "2015-10-06T16:34:42",
"processDefinitionId": "aProcessDefinitionId",
"processDefinitionKey": "aProcessDefinitionKey",
"processInstanceId": "aProcessInstanceId",
"tenantId": None,
"retries": 3,
"workerId": "aWorkerId",
"priority": 0,
"topicName": "createOrder",
"variables": {
"orderId": {
"type": "String",
"value": "3456",
"valueInfo": {}
}
}
}]
responses.add(responses.POST,
external_task_client.get_fetch_and_lock_url(),
status=HTTPStatus.OK,
json=resp_payload)
worker = ExternalTaskWorker(worker_id=0)
mock_action = mock.Mock()
task = ExternalTask({
"id": "anExternalTaskId",
"workerId": "aWorkerId",
"topicName": "createOrder"
})
mock_action.return_value = TaskResult.success(task=task,
global_variables={})
worker.fetch_and_execute("my_topic", mock_action)
self.assertEqual(2, mock_action.call_count)