async def execute_trigger(self, trigger, trigger_data):
""" Execute a trigger and ship the data """
logger.debug(
f"Echoing data from trigger: {trigger.name}-{self.workflow.execution_id}"
)
try:
result = trigger(trigger_data)
tmsg = NodeStatusMessage.success_from_node(
trigger, self.workflow.execution_id, result, parameters={})
await send_status_update(self.session, self.workflow.execution_id,
tmsg)
self.accumulator[trigger.id_] = result
self.in_process.pop(trigger.id_)
# TODO: can/should a trigger actually raise any exceptions?
except Exception as e:
logger.exception(
f"Worker received error for {trigger.name}-{self.workflow.execution_id}"
)
await send_status_update(
self.session, self.workflow.execution_id,
NodeStatusMessage.failure_from_node(trigger,
self.workflow.execution_id,
result=repr(e),
parameters={}))
async def abort(self):
logger.info(
f"Aborting workflow: {self.workflow.name} ({self.workflow.id_}) as {self.workflow.execution_id}"
)
[task.cancel() for task in self.scheduling_tasks]
self.results_getter_task.cancel()
self.execution_task.cancel()
# Try to cancel any outstanding actions
msgs = [
NodeStatusMessage.aborted_from_node(
action,
action.execution_id,
started_at=action.started_at,
parameters=(await self.dereference_params(action)))
for action in self.in_process.values()
]
message_tasks = [
send_status_update(self.session, self.workflow.execution_id, msg)
for msg in msgs
]
await asyncio.gather(*message_tasks, return_exceptions=True)
logger.info("Canceling outstanding tasks...")
await asyncio.gather(*self.scheduling_tasks,
*message_tasks,
self.results_getter_task,
self.execution_task,
return_exceptions=True)
logger.info(
f"Successfully aborted workflow: {self.workflow.name} ({self.workflow.id_}) as {self.workflow.execution_id}"
)
async def execute_parallel_action(self, node: Action, parameters):
schedule_tasks = []
actions = set()
action_to_parallel_map = {}
results = []
parallel_parameter = [p for p in node.parameters if p.parallelized]
unparallelized = list(set(node.parameters) - set(parallel_parameter))
for i, value in enumerate(parallel_parameter[0].value):
new_value = [value]
# params = node.append(array[i])
params = []
params.extend(unparallelized)
params.append(
Parameter(parallel_parameter[0].name,
value=new_value,
variant=ParameterVariant.STATIC_VALUE))
# params.append([parameter])
act = Action(node.name,
node.position,
node.app_name,
node.app_version,
f"{node.name}:shard_{value}",
node.priority,
parameters=params,
execution_id=node.execution_id)
actions.add(act.id_)
schedule_tasks.append(
asyncio.create_task(self.schedule_node(act, {}, {})))
action_to_parallel_map[act.id_] = new_value
self.parallel_in_process[act.id_] = act
# self.in_process.pop(node.id_)
exceptions = await asyncio.gather(*schedule_tasks,
return_exceptions=True)
while not actions.intersection(set(
self.parallel_accumulator.keys())) == actions:
await asyncio.sleep(0)
for a in actions:
contents = self.parallel_accumulator[a]
for individual in contents:
results.append(individual)
self.accumulator[node.id_] = results
# self.accumulator[node.id_] = [self.parallel_accumulator[a] for a in actions]
status = NodeStatusMessage.success_from_node(
node,
self.workflow.execution_id,
self.accumulator[node.id_],
parameters=parameters,
started_at=node.started_at)
await self.redis.xadd(self.results_stream,
{status.execution_id: message_dumps(status)})
async def execute_transform(self, transform, parents):
""" Execute an transform and ship its result """
logger.debug(
f"Attempting evaluation of: {transform.label}-{self.workflow.execution_id}"
)
try:
result = transform(
parents, self.accumulator) # run transform on parent's result
status = NodeStatusMessage.success_from_node(
transform,
self.workflow.execution_id,
result,
parameters={},
started_at=transform.started_at)
logger.info(
f"Transform {transform.label}-succeeded with result: {result}")
except TransformException as e:
logger.exception(
f"Worker received error for {transform.name}-{self.workflow.execution_id}"
)
aeval = Interpreter()
aeval(transform.transform)
if len(aeval.error) > 0:
error_tuple = (aeval.error[0]).get_error()
ret = error_tuple[0] + "(): " + error_tuple[1]
status = NodeStatusMessage.failure_from_node(
transform,
self.workflow.execution_id,
result=ret,
started_at=transform.started_at,
parameters={})
except Exception as e:
logger.exception(
f"Something bad happened in Transform evaluation: {e!r}")
return
# Send the status message through redis to ensure get_action_results completes it correctly
await self.redis.xadd(self.results_stream,
{status.execution_id: message_dumps(status)})
async def test_execute_action(app, action, redis):
assert await redis.lpop(action.execution_id) is None
await app.execute_action(action)
returned = (await redis.lpop(action.execution_id)).decode("utf-8")
returned = json.loads(returned)
result = 3
action_result = NodeStatusMessage.success_from_node(action, action.execution_id, result)
to_compare = message_dumps(action_result)
to_compare = json.loads(to_compare)
for key in returned.keys():
if key != "completed_at":
assert returned[key] == to_compare[key]
async def schedule_node(self, node, parents, children):
""" Waits until all dependencies of an action are met and then schedules the action """
logger.info(f"Scheduling {node.label}-{self.workflow.execution_id}...")
while not all(parent.id_ in self.accumulator
for parent in parents.values()):
logger.debug(
f"Node {node.label}-{self.workflow.execution_id} waiting for parents: {parents.values()}. "
f"Accumulator: {self.accumulator}")
await asyncio.sleep(1)
logger.info(
f"{node.label}-{self.workflow.execution_id} ready to execute.")
# node has more than one parent, check if both parent nodes have been cancelled
if len(parents) > 1:
count = 0
for parent in parents:
if parent in self.cancelled:
count += 1
if count == self.parent_map[node.id_]:
await self.cancel_subgraph(node)
if isinstance(node, Action):
if node.parallelized:
node.started_at = datetime.datetime.now()
params = await self.dereference_params(node)
await send_status_update(
self.session, self.workflow.execution_id,
NodeStatusMessage.executing_from_node(
node,
self.workflow.execution_id,
started_at=node.started_at,
parameters=params))
await send_status_update(
self.session, self.workflow.execution_id,
WorkflowStatusMessage.execution_continued(
self.workflow.execution_id,
self.workflow.id_,
self.workflow.name,
action_name=node.name,
app_name=node.app_name,
label=node.label))
asyncio.create_task(self.execute_parallel_action(node, params))
else:
group = f"{node.app_name}:{node.app_version}"
stream = f"{node.execution_id}:{group}"
try:
# The stream doesn't exist so lets create that and the app group
if len(await self.redis.keys(stream)) < 1:
await self.redis.xgroup_create(stream,
group,
mkstream=True)
# The stream exists but the group does not so lets just make the app group
if len(await self.redis.xinfo_groups(stream)) < 1:
await self.redis.xgroup_create(stream, group)
# Keep track of these for clean up later
self.streams.add(stream)
except aioredis.ReplyError as e:
logger.debug(f"Issue creating redis stream {e!r}")
params = await self.dereference_params(node)
node.started_at = datetime.datetime.now()
await send_status_update(
self.session, self.workflow.execution_id,
NodeStatusMessage.executing_from_node(
node,
self.workflow.execution_id,
started_at=node.started_at,
parameters=params))
await send_status_update(
self.session, self.workflow.execution_id,
WorkflowStatusMessage.execution_continued(
self.workflow.execution_id,
self.workflow.id_,
self.workflow.name,
action_name=node.name,
app_name=node.app_name,
label=node.label))
await self.redis.xadd(
stream, {node.execution_id: workflow_dumps(node)})
elif isinstance(node, Condition):
node.started_at = datetime.datetime.now()
await send_status_update(
self.session, self.workflow.execution_id,
NodeStatusMessage.executing_from_node(
node,
self.workflow.execution_id,
started_at=node.started_at,
parameters={}))
await send_status_update(
self.session, self.workflow.execution_id,
WorkflowStatusMessage.execution_continued(
self.workflow.execution_id,
self.workflow.id_,
self.workflow.name,
action_name=node.name,
app_name=node.app_name,
label=node.label))
await self.evaluate_condition(node, parents, children)
elif isinstance(node, Transform):
node.started_at = datetime.datetime.now()
await send_status_update(
self.session, self.workflow.execution_id,
NodeStatusMessage.executing_from_node(
node,
self.workflow.execution_id,
started_at=node.started_at,
parameters={}))
await send_status_update(
self.session, self.workflow.execution_id,
WorkflowStatusMessage.execution_continued(
self.workflow.execution_id,
self.workflow.id_,
self.workflow.name,
action_name=node.name,
app_name=node.app_name,
label=node.label))
await self.execute_transform(node, parents)
elif isinstance(node, Trigger):
trigger_stream = f"{self.workflow.execution_id}-{node.id_}:triggers"
msg = None
logger.info(
f"Trigger waiting in {self.workflow.name} at {node.label}-{self.workflow.execution_id}"
)
while not msg:
try:
with await self.redis as redis:
msg = await redis.xread_group(
static.REDIS_WORKFLOW_TRIGGERS_GROUP,
static.CONTAINER_ID,
streams=[trigger_stream],
count=1,
latest_ids=['>'])
logger.info(
f"Triggered {self.workflow.name} at {node.label}-{self.workflow.execution_id} "
f"with {msg}")
node.started_at = datetime.datetime.now()
await send_status_update(
self.session, self.workflow.execution_id,
NodeStatusMessage.executing_from_node(
node,
self.workflow.execution_id,
started_at=node.started_at))
await send_status_update(
self.session, self.workflow.execution_id,
WorkflowStatusMessage.execution_continued(
self.workflow.execution_id,
self.workflow.id_,
self.workflow.name,
action_name=node.name,
app_name=node.app_name,
label=node.label))
execution_id_trigger_message, stream, id_ = deref_stream_message(
msg)
execution_id, trigger_message = execution_id_trigger_message
trigger_message = message_loads(trigger_message)
await self.execute_trigger(node, trigger_message)
await self.redis.delete(trigger_stream)
except aioredis.errors.ReplyError as e:
logger.debug(
f"Stream {trigger_stream} doesn't exist. Attempting to create it..."
)
await self.redis.xgroup_create(
trigger_stream,
static.REDIS_WORKFLOW_TRIGGERS_GROUP,
mkstream=True,
latest_id='0')
# TODO: decide if we want pending action messages and uncomment this line
# await send_status_update(self.session, self.workflow.execution_id,
# NodeStatus.pending_from_node(node, workflow.execution_id))
logger.info(f"Scheduled {node}")
async def evaluate_condition(self, condition, parents, children):
"""
TODO: This will change when we implement a better UI element for it. For now, if an action is given a user
defined name like "Hello World", it would be referenced by the variable name "Hello_World" in the
conditional script. All whitespace in the action name is replaced by '_'. This is clearly problematic
if a user has an action named "Hello World" as well as "Hello_World". In this case, we cannot be sure
which is being referenced in the conditional and must raise an exception.
"""
logger.debug(
f"Attempting evaluation of: {condition.label}-{self.workflow.execution_id}"
)
try:
child_id = condition(parents, children, self.accumulator)
selected_node = children.pop(child_id)
status = NodeStatusMessage.success_from_node(
condition,
self.workflow.execution_id,
selected_node.name,
parameters={},
started_at=condition.started_at)
logger.info(
f"Condition selected node: {selected_node.label}-{self.workflow.execution_id}"
)
# We preemptively schedule all branches of execution so we must cancel all "false" branches here
for child in children.values():
if self.parent_map[child.id_] == 1:
await self.cancel_subgraph(child)
except ConditionException as e:
logger.exception(
f"Worker received error for {condition.name}-{self.workflow.execution_id}"
)
aeval = Interpreter()
aeval(condition.conditional)
if len(aeval.error) > 0:
error_tuple = (aeval.error[0]).get_error()
ret = error_tuple[0] + "(): " + error_tuple[1]
status = NodeStatusMessage.failure_from_node(
condition,
self.workflow.execution_id,
result=ret,
parameters={},
started_at=condition.started_at)
except KeyError as e:
logger.exception(
f"Worker received error for {condition.name}-{self.workflow.execution_id}"
)
status = NodeStatusMessage.failure_from_node(
condition,
self.workflow.execution_id,
result=
"ConditionError(): ensure that a non-parent node is selected and that the node name is not a string.",
parameters={},
started_at=condition.started_at)
except Exception as e:
logger.exception(
f"Something bad happened in Condition evaluation: {e!r}")
return
# Send the status message through redis to ensure get_action_results completes it correctly
await self.redis.xadd(self.results_stream,
{status.execution_id: message_dumps(status)})
async def execute_action(self, action: Action):
""" Execute an action, and push its result to Redis. """
# TODO: Is there a better way to do this?
self.logger.handlers[0].stream.execution_id = action.execution_id
self.logger.handlers[0].stream.workflow_id = action.workflow_id
self.logger.debug(
f"Attempting execution of: {action.label}-{action.execution_id}")
self.current_execution_id = action.execution_id
self.current_workflow_id = action.workflow_id
results_stream = f"{action.execution_id}:results"
if hasattr(self, action.name):
# Tell everyone we started execution
action.started_at = datetime.datetime.now()
start_action_msg = NodeStatusMessage.executing_from_node(
action, action.execution_id, started_at=action.started_at)
await self.redis.xadd(
results_stream,
{action.execution_id: message_dumps(start_action_msg)})
try:
func = getattr(self, action.name, None)
if callable(func):
if len(action.parameters) < 1:
result = await func()
else:
params = {}
for p in action.parameters:
if p.variant == ParameterVariant.GLOBAL:
key = config.get_from_file(
config.ENCRYPTION_KEY_PATH, 'rb')
params[p.name] = fernet_decrypt(key, p.value)
else:
params[p.name] = p.value
result = await func(**params)
action_result = NodeStatusMessage.success_from_node(
action,
action.execution_id,
result=result,
started_at=action.started_at)
self.logger.debug(
f"Executed {action.label}-{action.execution_id} "
f"with result: {result}")
else:
self.logger.error(
f"App {self.__class__.__name__}.{action.name} is not callable"
)
action_result = NodeStatusMessage.failure_from_node(
action,
action.execution_id,
result="Action not callable",
started_at=action.started_at)
except Exception as e:
self.logger.exception(
f"Failed to execute {action.label}-{action.execution_id}")
action_result = NodeStatusMessage.failure_from_node(
action,
action.execution_id,
result=repr(e),
started_at=action.started_at)
else:
self.logger.error(
f"App {self.__class__.__name__} has no method {action.name}")
action_result = NodeStatusMessage.failure_from_node(
action,
action.execution_id,
result="Action does not exist",
started_at=action.started_at)
await self.redis.xadd(
results_stream,
{action.execution_id: message_dumps(action_result)})