async def execute_trigger(self, trigger, trigger_data):
""" Execute a trigger and ship the data """
logger.debug(
f"Echoing data from trigger: {trigger.name}-{self.workflow.execution_id}"
)
try:
result = trigger(trigger_data)
tmsg = NodeStatusMessage.success_from_node(
trigger, self.workflow.execution_id, result, parameters={})
await send_status_update(self.session, self.workflow.execution_id,
tmsg)
self.accumulator[trigger.id_] = result
self.in_process.pop(trigger.id_)
# TODO: can/should a trigger actually raise any exceptions?
except Exception as e:
logger.exception(
f"Worker received error for {trigger.name}-{self.workflow.execution_id}"
)
await send_status_update(
self.session, self.workflow.execution_id,
NodeStatusMessage.failure_from_node(trigger,
self.workflow.execution_id,
result=repr(e),
parameters={}))
async def execute_parallel_action(self, node: Action, parameters):
schedule_tasks = []
actions = set()
action_to_parallel_map = {}
results = []
parallel_parameter = [p for p in node.parameters if p.parallelized]
unparallelized = list(set(node.parameters) - set(parallel_parameter))
for i, value in enumerate(parallel_parameter[0].value):
new_value = [value]
# params = node.append(array[i])
params = []
params.extend(unparallelized)
params.append(
Parameter(parallel_parameter[0].name,
value=new_value,
variant=ParameterVariant.STATIC_VALUE))
# params.append([parameter])
act = Action(node.name,
node.position,
node.app_name,
node.app_version,
f"{node.name}:shard_{value}",
node.priority,
parameters=params,
execution_id=node.execution_id)
actions.add(act.id_)
schedule_tasks.append(
asyncio.create_task(self.schedule_node(act, {}, {})))
action_to_parallel_map[act.id_] = new_value
self.parallel_in_process[act.id_] = act
# self.in_process.pop(node.id_)
exceptions = await asyncio.gather(*schedule_tasks,
return_exceptions=True)
while not actions.intersection(set(
self.parallel_accumulator.keys())) == actions:
await asyncio.sleep(0)
for a in actions:
contents = self.parallel_accumulator[a]
for individual in contents:
results.append(individual)
self.accumulator[node.id_] = results
# self.accumulator[node.id_] = [self.parallel_accumulator[a] for a in actions]
status = NodeStatusMessage.success_from_node(
node,
self.workflow.execution_id,
self.accumulator[node.id_],
parameters=parameters,
started_at=node.started_at)
await self.redis.xadd(self.results_stream,
{status.execution_id: message_dumps(status)})
async def test_execute_action(app, action, redis):
assert await redis.lpop(action.execution_id) is None
await app.execute_action(action)
returned = (await redis.lpop(action.execution_id)).decode("utf-8")
returned = json.loads(returned)
result = 3
action_result = NodeStatusMessage.success_from_node(action, action.execution_id, result)
to_compare = message_dumps(action_result)
to_compare = json.loads(to_compare)
for key in returned.keys():
if key != "completed_at":
assert returned[key] == to_compare[key]
async def execute_transform(self, transform, parents):
""" Execute an transform and ship its result """
logger.debug(
f"Attempting evaluation of: {transform.label}-{self.workflow.execution_id}"
)
try:
result = transform(
parents, self.accumulator) # run transform on parent's result
status = NodeStatusMessage.success_from_node(
transform,
self.workflow.execution_id,
result,
parameters={},
started_at=transform.started_at)
logger.info(
f"Transform {transform.label}-succeeded with result: {result}")
except TransformException as e:
logger.exception(
f"Worker received error for {transform.name}-{self.workflow.execution_id}"
)
aeval = Interpreter()
aeval(transform.transform)
if len(aeval.error) > 0:
error_tuple = (aeval.error[0]).get_error()
ret = error_tuple[0] + "(): " + error_tuple[1]
status = NodeStatusMessage.failure_from_node(
transform,
self.workflow.execution_id,
result=ret,
started_at=transform.started_at,
parameters={})
except Exception as e:
logger.exception(
f"Something bad happened in Transform evaluation: {e!r}")
return
# Send the status message through redis to ensure get_action_results completes it correctly
await self.redis.xadd(self.results_stream,
{status.execution_id: message_dumps(status)})
async def evaluate_condition(self, condition, parents, children):
"""
TODO: This will change when we implement a better UI element for it. For now, if an action is given a user
defined name like "Hello World", it would be referenced by the variable name "Hello_World" in the
conditional script. All whitespace in the action name is replaced by '_'. This is clearly problematic
if a user has an action named "Hello World" as well as "Hello_World". In this case, we cannot be sure
which is being referenced in the conditional and must raise an exception.
"""
logger.debug(
f"Attempting evaluation of: {condition.label}-{self.workflow.execution_id}"
)
try:
child_id = condition(parents, children, self.accumulator)
selected_node = children.pop(child_id)
status = NodeStatusMessage.success_from_node(
condition,
self.workflow.execution_id,
selected_node.name,
parameters={},
started_at=condition.started_at)
logger.info(
f"Condition selected node: {selected_node.label}-{self.workflow.execution_id}"
)
# We preemptively schedule all branches of execution so we must cancel all "false" branches here
for child in children.values():
if self.parent_map[child.id_] == 1:
await self.cancel_subgraph(child)
except ConditionException as e:
logger.exception(
f"Worker received error for {condition.name}-{self.workflow.execution_id}"
)
aeval = Interpreter()
aeval(condition.conditional)
if len(aeval.error) > 0:
error_tuple = (aeval.error[0]).get_error()
ret = error_tuple[0] + "(): " + error_tuple[1]
status = NodeStatusMessage.failure_from_node(
condition,
self.workflow.execution_id,
result=ret,
parameters={},
started_at=condition.started_at)
except KeyError as e:
logger.exception(
f"Worker received error for {condition.name}-{self.workflow.execution_id}"
)
status = NodeStatusMessage.failure_from_node(
condition,
self.workflow.execution_id,
result=
"ConditionError(): ensure that a non-parent node is selected and that the node name is not a string.",
parameters={},
started_at=condition.started_at)
except Exception as e:
logger.exception(
f"Something bad happened in Condition evaluation: {e!r}")
return
# Send the status message through redis to ensure get_action_results completes it correctly
await self.redis.xadd(self.results_stream,
{status.execution_id: message_dumps(status)})
async def execute_action(self, action: Action):
""" Execute an action, and push its result to Redis. """
# TODO: Is there a better way to do this?
self.logger.handlers[0].stream.execution_id = action.execution_id
self.logger.handlers[0].stream.workflow_id = action.workflow_id
self.logger.debug(
f"Attempting execution of: {action.label}-{action.execution_id}")
self.current_execution_id = action.execution_id
self.current_workflow_id = action.workflow_id
results_stream = f"{action.execution_id}:results"
if hasattr(self, action.name):
# Tell everyone we started execution
action.started_at = datetime.datetime.now()
start_action_msg = NodeStatusMessage.executing_from_node(
action, action.execution_id, started_at=action.started_at)
await self.redis.xadd(
results_stream,
{action.execution_id: message_dumps(start_action_msg)})
try:
func = getattr(self, action.name, None)
if callable(func):
if len(action.parameters) < 1:
result = await func()
else:
params = {}
for p in action.parameters:
if p.variant == ParameterVariant.GLOBAL:
key = config.get_from_file(
config.ENCRYPTION_KEY_PATH, 'rb')
params[p.name] = fernet_decrypt(key, p.value)
else:
params[p.name] = p.value
result = await func(**params)
action_result = NodeStatusMessage.success_from_node(
action,
action.execution_id,
result=result,
started_at=action.started_at)
self.logger.debug(
f"Executed {action.label}-{action.execution_id} "
f"with result: {result}")
else:
self.logger.error(
f"App {self.__class__.__name__}.{action.name} is not callable"
)
action_result = NodeStatusMessage.failure_from_node(
action,
action.execution_id,
result="Action not callable",
started_at=action.started_at)
except Exception as e:
self.logger.exception(
f"Failed to execute {action.label}-{action.execution_id}")
action_result = NodeStatusMessage.failure_from_node(
action,
action.execution_id,
result=repr(e),
started_at=action.started_at)
else:
self.logger.error(
f"App {self.__class__.__name__} has no method {action.name}")
action_result = NodeStatusMessage.failure_from_node(
action,
action.execution_id,
result="Action does not exist",
started_at=action.started_at)
await self.redis.xadd(
results_stream,
{action.execution_id: message_dumps(action_result)})