async def execute_trigger(self, trigger, trigger_data): """ Execute a trigger and ship the data """ logger.debug( f"Echoing data from trigger: {trigger.name}-{self.workflow.execution_id}" ) try: result = trigger(trigger_data) tmsg = NodeStatusMessage.success_from_node( trigger, self.workflow.execution_id, result, parameters={}) await send_status_update(self.session, self.workflow.execution_id, tmsg) self.accumulator[trigger.id_] = result self.in_process.pop(trigger.id_) # TODO: can/should a trigger actually raise any exceptions? except Exception as e: logger.exception( f"Worker received error for {trigger.name}-{self.workflow.execution_id}" ) await send_status_update( self.session, self.workflow.execution_id, NodeStatusMessage.failure_from_node(trigger, self.workflow.execution_id, result=repr(e), parameters={}))
async def abort(self): logger.info( f"Aborting workflow: {self.workflow.name} ({self.workflow.id_}) as {self.workflow.execution_id}" ) [task.cancel() for task in self.scheduling_tasks] self.results_getter_task.cancel() self.execution_task.cancel() # Try to cancel any outstanding actions msgs = [ NodeStatusMessage.aborted_from_node( action, action.execution_id, started_at=action.started_at, parameters=(await self.dereference_params(action))) for action in self.in_process.values() ] message_tasks = [ send_status_update(self.session, self.workflow.execution_id, msg) for msg in msgs ] await asyncio.gather(*message_tasks, return_exceptions=True) logger.info("Canceling outstanding tasks...") await asyncio.gather(*self.scheduling_tasks, *message_tasks, self.results_getter_task, self.execution_task, return_exceptions=True) logger.info( f"Successfully aborted workflow: {self.workflow.name} ({self.workflow.id_}) as {self.workflow.execution_id}" )
async def execute_parallel_action(self, node: Action, parameters): schedule_tasks = [] actions = set() action_to_parallel_map = {} results = [] parallel_parameter = [p for p in node.parameters if p.parallelized] unparallelized = list(set(node.parameters) - set(parallel_parameter)) for i, value in enumerate(parallel_parameter[0].value): new_value = [value] # params = node.append(array[i]) params = [] params.extend(unparallelized) params.append( Parameter(parallel_parameter[0].name, value=new_value, variant=ParameterVariant.STATIC_VALUE)) # params.append([parameter]) act = Action(node.name, node.position, node.app_name, node.app_version, f"{node.name}:shard_{value}", node.priority, parameters=params, execution_id=node.execution_id) actions.add(act.id_) schedule_tasks.append( asyncio.create_task(self.schedule_node(act, {}, {}))) action_to_parallel_map[act.id_] = new_value self.parallel_in_process[act.id_] = act # self.in_process.pop(node.id_) exceptions = await asyncio.gather(*schedule_tasks, return_exceptions=True) while not actions.intersection(set( self.parallel_accumulator.keys())) == actions: await asyncio.sleep(0) for a in actions: contents = self.parallel_accumulator[a] for individual in contents: results.append(individual) self.accumulator[node.id_] = results # self.accumulator[node.id_] = [self.parallel_accumulator[a] for a in actions] status = NodeStatusMessage.success_from_node( node, self.workflow.execution_id, self.accumulator[node.id_], parameters=parameters, started_at=node.started_at) await self.redis.xadd(self.results_stream, {status.execution_id: message_dumps(status)})
async def execute_transform(self, transform, parents): """ Execute an transform and ship its result """ logger.debug( f"Attempting evaluation of: {transform.label}-{self.workflow.execution_id}" ) try: result = transform( parents, self.accumulator) # run transform on parent's result status = NodeStatusMessage.success_from_node( transform, self.workflow.execution_id, result, parameters={}, started_at=transform.started_at) logger.info( f"Transform {transform.label}-succeeded with result: {result}") except TransformException as e: logger.exception( f"Worker received error for {transform.name}-{self.workflow.execution_id}" ) aeval = Interpreter() aeval(transform.transform) if len(aeval.error) > 0: error_tuple = (aeval.error[0]).get_error() ret = error_tuple[0] + "(): " + error_tuple[1] status = NodeStatusMessage.failure_from_node( transform, self.workflow.execution_id, result=ret, started_at=transform.started_at, parameters={}) except Exception as e: logger.exception( f"Something bad happened in Transform evaluation: {e!r}") return # Send the status message through redis to ensure get_action_results completes it correctly await self.redis.xadd(self.results_stream, {status.execution_id: message_dumps(status)})
async def test_execute_action(app, action, redis): assert await redis.lpop(action.execution_id) is None await app.execute_action(action) returned = (await redis.lpop(action.execution_id)).decode("utf-8") returned = json.loads(returned) result = 3 action_result = NodeStatusMessage.success_from_node(action, action.execution_id, result) to_compare = message_dumps(action_result) to_compare = json.loads(to_compare) for key in returned.keys(): if key != "completed_at": assert returned[key] == to_compare[key]
async def schedule_node(self, node, parents, children): """ Waits until all dependencies of an action are met and then schedules the action """ logger.info(f"Scheduling {node.label}-{self.workflow.execution_id}...") while not all(parent.id_ in self.accumulator for parent in parents.values()): logger.debug( f"Node {node.label}-{self.workflow.execution_id} waiting for parents: {parents.values()}. " f"Accumulator: {self.accumulator}") await asyncio.sleep(1) logger.info( f"{node.label}-{self.workflow.execution_id} ready to execute.") # node has more than one parent, check if both parent nodes have been cancelled if len(parents) > 1: count = 0 for parent in parents: if parent in self.cancelled: count += 1 if count == self.parent_map[node.id_]: await self.cancel_subgraph(node) if isinstance(node, Action): if node.parallelized: node.started_at = datetime.datetime.now() params = await self.dereference_params(node) await send_status_update( self.session, self.workflow.execution_id, NodeStatusMessage.executing_from_node( node, self.workflow.execution_id, started_at=node.started_at, parameters=params)) await send_status_update( self.session, self.workflow.execution_id, WorkflowStatusMessage.execution_continued( self.workflow.execution_id, self.workflow.id_, self.workflow.name, action_name=node.name, app_name=node.app_name, label=node.label)) asyncio.create_task(self.execute_parallel_action(node, params)) else: group = f"{node.app_name}:{node.app_version}" stream = f"{node.execution_id}:{group}" try: # The stream doesn't exist so lets create that and the app group if len(await self.redis.keys(stream)) < 1: await self.redis.xgroup_create(stream, group, mkstream=True) # The stream exists but the group does not so lets just make the app group if len(await self.redis.xinfo_groups(stream)) < 1: await self.redis.xgroup_create(stream, group) # Keep track of these for clean up later self.streams.add(stream) except aioredis.ReplyError as e: logger.debug(f"Issue creating redis stream {e!r}") params = await self.dereference_params(node) node.started_at = datetime.datetime.now() await send_status_update( self.session, self.workflow.execution_id, NodeStatusMessage.executing_from_node( node, self.workflow.execution_id, started_at=node.started_at, parameters=params)) await send_status_update( self.session, self.workflow.execution_id, WorkflowStatusMessage.execution_continued( self.workflow.execution_id, self.workflow.id_, self.workflow.name, action_name=node.name, app_name=node.app_name, label=node.label)) await self.redis.xadd( stream, {node.execution_id: workflow_dumps(node)}) elif isinstance(node, Condition): node.started_at = datetime.datetime.now() await send_status_update( self.session, self.workflow.execution_id, NodeStatusMessage.executing_from_node( node, self.workflow.execution_id, started_at=node.started_at, parameters={})) await send_status_update( self.session, self.workflow.execution_id, WorkflowStatusMessage.execution_continued( self.workflow.execution_id, self.workflow.id_, self.workflow.name, action_name=node.name, app_name=node.app_name, label=node.label)) await self.evaluate_condition(node, parents, children) elif isinstance(node, Transform): node.started_at = datetime.datetime.now() await send_status_update( self.session, self.workflow.execution_id, NodeStatusMessage.executing_from_node( node, self.workflow.execution_id, started_at=node.started_at, parameters={})) await send_status_update( self.session, self.workflow.execution_id, WorkflowStatusMessage.execution_continued( self.workflow.execution_id, self.workflow.id_, self.workflow.name, action_name=node.name, app_name=node.app_name, label=node.label)) await self.execute_transform(node, parents) elif isinstance(node, Trigger): trigger_stream = f"{self.workflow.execution_id}-{node.id_}:triggers" msg = None logger.info( f"Trigger waiting in {self.workflow.name} at {node.label}-{self.workflow.execution_id}" ) while not msg: try: with await self.redis as redis: msg = await redis.xread_group( static.REDIS_WORKFLOW_TRIGGERS_GROUP, static.CONTAINER_ID, streams=[trigger_stream], count=1, latest_ids=['>']) logger.info( f"Triggered {self.workflow.name} at {node.label}-{self.workflow.execution_id} " f"with {msg}") node.started_at = datetime.datetime.now() await send_status_update( self.session, self.workflow.execution_id, NodeStatusMessage.executing_from_node( node, self.workflow.execution_id, started_at=node.started_at)) await send_status_update( self.session, self.workflow.execution_id, WorkflowStatusMessage.execution_continued( self.workflow.execution_id, self.workflow.id_, self.workflow.name, action_name=node.name, app_name=node.app_name, label=node.label)) execution_id_trigger_message, stream, id_ = deref_stream_message( msg) execution_id, trigger_message = execution_id_trigger_message trigger_message = message_loads(trigger_message) await self.execute_trigger(node, trigger_message) await self.redis.delete(trigger_stream) except aioredis.errors.ReplyError as e: logger.debug( f"Stream {trigger_stream} doesn't exist. Attempting to create it..." ) await self.redis.xgroup_create( trigger_stream, static.REDIS_WORKFLOW_TRIGGERS_GROUP, mkstream=True, latest_id='0') # TODO: decide if we want pending action messages and uncomment this line # await send_status_update(self.session, self.workflow.execution_id, # NodeStatus.pending_from_node(node, workflow.execution_id)) logger.info(f"Scheduled {node}")
async def evaluate_condition(self, condition, parents, children): """ TODO: This will change when we implement a better UI element for it. For now, if an action is given a user defined name like "Hello World", it would be referenced by the variable name "Hello_World" in the conditional script. All whitespace in the action name is replaced by '_'. This is clearly problematic if a user has an action named "Hello World" as well as "Hello_World". In this case, we cannot be sure which is being referenced in the conditional and must raise an exception. """ logger.debug( f"Attempting evaluation of: {condition.label}-{self.workflow.execution_id}" ) try: child_id = condition(parents, children, self.accumulator) selected_node = children.pop(child_id) status = NodeStatusMessage.success_from_node( condition, self.workflow.execution_id, selected_node.name, parameters={}, started_at=condition.started_at) logger.info( f"Condition selected node: {selected_node.label}-{self.workflow.execution_id}" ) # We preemptively schedule all branches of execution so we must cancel all "false" branches here for child in children.values(): if self.parent_map[child.id_] == 1: await self.cancel_subgraph(child) except ConditionException as e: logger.exception( f"Worker received error for {condition.name}-{self.workflow.execution_id}" ) aeval = Interpreter() aeval(condition.conditional) if len(aeval.error) > 0: error_tuple = (aeval.error[0]).get_error() ret = error_tuple[0] + "(): " + error_tuple[1] status = NodeStatusMessage.failure_from_node( condition, self.workflow.execution_id, result=ret, parameters={}, started_at=condition.started_at) except KeyError as e: logger.exception( f"Worker received error for {condition.name}-{self.workflow.execution_id}" ) status = NodeStatusMessage.failure_from_node( condition, self.workflow.execution_id, result= "ConditionError(): ensure that a non-parent node is selected and that the node name is not a string.", parameters={}, started_at=condition.started_at) except Exception as e: logger.exception( f"Something bad happened in Condition evaluation: {e!r}") return # Send the status message through redis to ensure get_action_results completes it correctly await self.redis.xadd(self.results_stream, {status.execution_id: message_dumps(status)})
async def execute_action(self, action: Action): """ Execute an action, and push its result to Redis. """ # TODO: Is there a better way to do this? self.logger.handlers[0].stream.execution_id = action.execution_id self.logger.handlers[0].stream.workflow_id = action.workflow_id self.logger.debug( f"Attempting execution of: {action.label}-{action.execution_id}") self.current_execution_id = action.execution_id self.current_workflow_id = action.workflow_id results_stream = f"{action.execution_id}:results" if hasattr(self, action.name): # Tell everyone we started execution action.started_at = datetime.datetime.now() start_action_msg = NodeStatusMessage.executing_from_node( action, action.execution_id, started_at=action.started_at) await self.redis.xadd( results_stream, {action.execution_id: message_dumps(start_action_msg)}) try: func = getattr(self, action.name, None) if callable(func): if len(action.parameters) < 1: result = await func() else: params = {} for p in action.parameters: if p.variant == ParameterVariant.GLOBAL: key = config.get_from_file( config.ENCRYPTION_KEY_PATH, 'rb') params[p.name] = fernet_decrypt(key, p.value) else: params[p.name] = p.value result = await func(**params) action_result = NodeStatusMessage.success_from_node( action, action.execution_id, result=result, started_at=action.started_at) self.logger.debug( f"Executed {action.label}-{action.execution_id} " f"with result: {result}") else: self.logger.error( f"App {self.__class__.__name__}.{action.name} is not callable" ) action_result = NodeStatusMessage.failure_from_node( action, action.execution_id, result="Action not callable", started_at=action.started_at) except Exception as e: self.logger.exception( f"Failed to execute {action.label}-{action.execution_id}") action_result = NodeStatusMessage.failure_from_node( action, action.execution_id, result=repr(e), started_at=action.started_at) else: self.logger.error( f"App {self.__class__.__name__} has no method {action.name}") action_result = NodeStatusMessage.failure_from_node( action, action.execution_id, result="Action does not exist", started_at=action.started_at) await self.redis.xadd( results_stream, {action.execution_id: message_dumps(action_result)})