async def execute_action(self, action: Action): """ Execute an action and ship its result """ self.logger.debug( f"Attempting execution of: {action.label}-{action.execution_id}") self.console_logger.handlers[0].stream.set_channel( f"{action.execution_id}:console") if hasattr(self, action.name): start_action_msg = NodeStatus.executing_from_node( action, action.execution_id) await self.redis.lpush(action.execution_id, message_dumps(start_action_msg)) try: func = getattr(self, action.name, None) if callable(func): if len(action.parameters) < 1: result = await func() else: result = await func( **{p.name: p.value for p in action.parameters}) action_result = NodeStatus.success_from_node( action, action.execution_id, result) self.logger.debug( f"Executed {action.label}-{action.id_} with result: {result}" ) else: self.logger.error( f"App {self.__class__.__name__}.{action.name} is not callable" ) action_result = NodeStatus.failure_from_node( action, action.execution_id, error="Action not callable") except Exception as e: action_result = NodeStatus.failure_from_node( action, action.execution_id, error=repr(e)) self.logger.exception( f"Failed to execute {action.label}-{action.id_}") await self.redis.lpush(action.execution_id, message_dumps(action_result)) else: self.logger.error( f"App {self.__class__.__name__} has no method {action.name}") action_result = NodeStatus.failure_from_node( action, action.execution_id, error="Action does not exist") await self.redis.lpush(action.execution_id, message_dumps(action_result))
async def send_status_update(session, execution_id, message, headers=None): import aiohttp """ Forms and sends a JSONPatch message to the api_gateway to update the status of an action or workflow """ if message is None: return None patches = get_patches(message) if len(patches) < 1: raise ValueError( f"Attempting to send improper message type: {type(message)}") params = {"event": message.status.value} url = f"{config.API_GATEWAY_URI}/walkoff/api/internal/workflowstatus/{execution_id}" headers, token = await get_walkoff_auth_header(session) headers["content-type"] = "application/json" try: async with session.patch(url, data=message_dumps(patches), params=params, headers=headers, timeout=5) as resp: if resp.content_type == "application/json": results = await resp.json() logger.debug(f"API-Gateway status update response: {results}") return results except aiohttp.ClientConnectionError as e: logger.error(f"Could not send status message to {url}: {e!r}") except Exception as e: logger.error(f"Unknown error while sending message to {url}: {e!r}")
async def execute_parallel_action(self, node: Action, parameters): schedule_tasks = [] actions = set() action_to_parallel_map = {} results = [] parallel_parameter = [p for p in node.parameters if p.parallelized] unparallelized = list(set(node.parameters) - set(parallel_parameter)) for i, value in enumerate(parallel_parameter[0].value): new_value = [value] # params = node.append(array[i]) params = [] params.extend(unparallelized) params.append( Parameter(parallel_parameter[0].name, value=new_value, variant=ParameterVariant.STATIC_VALUE)) # params.append([parameter]) act = Action(node.name, node.position, node.app_name, node.app_version, f"{node.name}:shard_{value}", node.priority, parameters=params, execution_id=node.execution_id) actions.add(act.id_) schedule_tasks.append( asyncio.create_task(self.schedule_node(act, {}, {}))) action_to_parallel_map[act.id_] = new_value self.parallel_in_process[act.id_] = act # self.in_process.pop(node.id_) exceptions = await asyncio.gather(*schedule_tasks, return_exceptions=True) while not actions.intersection(set( self.parallel_accumulator.keys())) == actions: await asyncio.sleep(0) for a in actions: contents = self.parallel_accumulator[a] for individual in contents: results.append(individual) self.accumulator[node.id_] = results # self.accumulator[node.id_] = [self.parallel_accumulator[a] for a in actions] status = NodeStatusMessage.success_from_node( node, self.workflow.execution_id, self.accumulator[node.id_], parameters=parameters, started_at=node.started_at) await self.redis.xadd(self.results_stream, {status.execution_id: message_dumps(status)})
async def send_status_update(redis, execution_id, workflow_id, message): """ Forms and sends a JSONPatch message to the api_gateway to update the status of an action or workflow """ if message is None: return None patches = { "execution_id": execution_id, "workflow_id": workflow_id, "message": message_dumps(get_patches(message)), "type": "workflow" if type(message) is WorkflowStatusMessage else "node" } # try: logger.info(f"Sending result {patches}") await redis.lpush(static.REDIS_RESULTS_QUEUE, json.dumps(patches))
async def test_execute_action(app, action, redis): assert await redis.lpop(action.execution_id) is None await app.execute_action(action) returned = (await redis.lpop(action.execution_id)).decode("utf-8") returned = json.loads(returned) result = 3 action_result = NodeStatusMessage.success_from_node(action, action.execution_id, result) to_compare = message_dumps(action_result) to_compare = json.loads(to_compare) for key in returned.keys(): if key != "completed_at": assert returned[key] == to_compare[key]
async def send_message(self, message: Union[NodeStatus, WorkflowStatus]): """ Forms and sends a JSONPatch message to the api_gateway to update the status of an action or workflow """ patches = self.get_patches(message) if patches is None: raise ValueError( f"Attempting to send improper message type: {type(message)}") data = message_dumps(patches) params = {"event": message.status.value} url = f"{config['WORKER']['api_gateway_uri']}/iapi/workflowstatus/{self.workflow.execution_id}" try: async with self.session.patch(url, data=data, params=params) as resp: return resp.json(loads=message_loads) except aiohttp.ClientConnectionError as e: logger.error(f"Could not send status message to {url}: {e!r}")
async def execute_transform(self, transform, parents): """ Execute an transform and ship its result """ logger.debug( f"Attempting evaluation of: {transform.label}-{self.workflow.execution_id}" ) try: result = transform( parents, self.accumulator) # run transform on parent's result status = NodeStatusMessage.success_from_node( transform, self.workflow.execution_id, result, parameters={}, started_at=transform.started_at) logger.info( f"Transform {transform.label}-succeeded with result: {result}") except TransformException as e: logger.exception( f"Worker received error for {transform.name}-{self.workflow.execution_id}" ) aeval = Interpreter() aeval(transform.transform) if len(aeval.error) > 0: error_tuple = (aeval.error[0]).get_error() ret = error_tuple[0] + "(): " + error_tuple[1] status = NodeStatusMessage.failure_from_node( transform, self.workflow.execution_id, result=ret, started_at=transform.started_at, parameters={}) except Exception as e: logger.exception( f"Something bad happened in Transform evaluation: {e!r}") return # Send the status message through redis to ensure get_action_results completes it correctly await self.redis.xadd(self.results_stream, {status.execution_id: message_dumps(status)})
async def control_workflow( request: Request, execution, workflow_to_control: ControlWorkflow, workflow_status_col: AsyncIOMotorCollection = Depends(get_mongo_c)): """ Pause, resume, or abort a workflow currently executing in WALKOFF. """ execution = await mongo_helpers.get_item(workflow_status_col, WorkflowStatus, execution, id_key="execution_id") walkoff_db = get_mongo_d(request) workflow_col = walkoff_db.workflows curr_user_id = await get_jwt_identity(request) workflow_id = execution.workflow_id data = dict(workflow_to_control) status = data['status'] workflow: WorkflowModel = await mongo_helpers.get_item( workflow_col, WorkflowModel, workflow_id) # workflow = workflow_getter(execution.workflow_id, workflow_status_col) # The resource factory returns the WorkflowStatus model but we want the string of the execution ID execution_id = str(execution.execution_id) to_execute = await auth_check(workflow, curr_user_id, "execute", walkoff_db=walkoff_db) # TODO: add in pause/resume here. Workers need to store and recover state for this if to_execute: if status.lower() == 'abort': logger.info( f"User '{(await get_jwt_claims(request)).get('username', None)}' aborting workflow: {execution_id}" ) message = { "execution_id": execution_id, "status": status, "workflow": dict(workflow) } async with connect_to_aioredis_pool(config.REDIS_URI) as conn: await conn.smove(static.REDIS_PENDING_WORKFLOWS, static.REDIS_ABORTING_WORKFLOWS, execution_id) await conn.xadd(static.REDIS_WORKFLOW_CONTROL, message) return None, HTTPStatus.NO_CONTENT elif status.lower() == 'trigger': if execution.status not in (StatusEnum.PENDING, StatusEnum.EXECUTING, StatusEnum.AWAITING_DATA): raise InvalidInputException( "workflow", "trigger", execution_id, errors=[ "Workflow must be in a running state to accept triggers." ]) trigger_id = data.get('trigger_id') if not trigger_id: raise InvalidInputException( "workflow", "trigger", execution_id, errors=[ "ID of the trigger must be specified in trigger_id." ]) seen = False for trigger in workflow.triggers: if str(trigger.id_) == trigger_id: seen = True if not seen: raise InvalidInputException( "workflow", "trigger", execution_id, errors=[ f"trigger_id {trigger_id} was not found in this workflow." ]) trigger_stream = f"{execution_id}-{trigger_id}:triggers" try: async with connect_to_aioredis_pool(config.REDIS_URI) as conn: info = await conn.xinfo_stream(trigger_stream) stream_length = info["length"] except Exception: stream_length = 0 if stream_length > 0: return InvalidInputException( "workflow", "trigger", execution_id, errors=[f"This trigger has already received data."]) trigger_data = data.get('trigger_data') logger.info( f"User '{(await get_jwt_claims(request)).get('username', None)}' triggering workflow: {execution_id} at trigger " f"{trigger_id} with data {trigger_data}") async with connect_to_aioredis_pool(config.REDIS_URI) as conn: await conn.xadd(trigger_stream, { execution_id: message_dumps({"trigger_data": trigger_data}) }) return ({"trigger_stream": trigger_stream}) else: return None
async def evaluate_condition(self, condition, parents, children): """ TODO: This will change when we implement a better UI element for it. For now, if an action is given a user defined name like "Hello World", it would be referenced by the variable name "Hello_World" in the conditional script. All whitespace in the action name is replaced by '_'. This is clearly problematic if a user has an action named "Hello World" as well as "Hello_World". In this case, we cannot be sure which is being referenced in the conditional and must raise an exception. """ logger.debug( f"Attempting evaluation of: {condition.label}-{self.workflow.execution_id}" ) try: child_id = condition(parents, children, self.accumulator) selected_node = children.pop(child_id) status = NodeStatusMessage.success_from_node( condition, self.workflow.execution_id, selected_node.name, parameters={}, started_at=condition.started_at) logger.info( f"Condition selected node: {selected_node.label}-{self.workflow.execution_id}" ) # We preemptively schedule all branches of execution so we must cancel all "false" branches here for child in children.values(): if self.parent_map[child.id_] == 1: await self.cancel_subgraph(child) except ConditionException as e: logger.exception( f"Worker received error for {condition.name}-{self.workflow.execution_id}" ) aeval = Interpreter() aeval(condition.conditional) if len(aeval.error) > 0: error_tuple = (aeval.error[0]).get_error() ret = error_tuple[0] + "(): " + error_tuple[1] status = NodeStatusMessage.failure_from_node( condition, self.workflow.execution_id, result=ret, parameters={}, started_at=condition.started_at) except KeyError as e: logger.exception( f"Worker received error for {condition.name}-{self.workflow.execution_id}" ) status = NodeStatusMessage.failure_from_node( condition, self.workflow.execution_id, result= "ConditionError(): ensure that a non-parent node is selected and that the node name is not a string.", parameters={}, started_at=condition.started_at) except Exception as e: logger.exception( f"Something bad happened in Condition evaluation: {e!r}") return # Send the status message through redis to ensure get_action_results completes it correctly await self.redis.xadd(self.results_stream, {status.execution_id: message_dumps(status)})
async def execute_action(self, action: Action): """ Execute an action, and push its result to Redis. """ # TODO: Is there a better way to do this? self.logger.handlers[0].stream.execution_id = action.execution_id self.logger.handlers[0].stream.workflow_id = action.workflow_id self.logger.debug( f"Attempting execution of: {action.label}-{action.execution_id}") self.current_execution_id = action.execution_id self.current_workflow_id = action.workflow_id results_stream = f"{action.execution_id}:results" if hasattr(self, action.name): # Tell everyone we started execution action.started_at = datetime.datetime.now() start_action_msg = NodeStatusMessage.executing_from_node( action, action.execution_id, started_at=action.started_at) await self.redis.xadd( results_stream, {action.execution_id: message_dumps(start_action_msg)}) try: func = getattr(self, action.name, None) if callable(func): if len(action.parameters) < 1: result = await func() else: params = {} for p in action.parameters: if p.variant == ParameterVariant.GLOBAL: key = config.get_from_file( config.ENCRYPTION_KEY_PATH, 'rb') params[p.name] = fernet_decrypt(key, p.value) else: params[p.name] = p.value result = await func(**params) action_result = NodeStatusMessage.success_from_node( action, action.execution_id, result=result, started_at=action.started_at) self.logger.debug( f"Executed {action.label}-{action.execution_id} " f"with result: {result}") else: self.logger.error( f"App {self.__class__.__name__}.{action.name} is not callable" ) action_result = NodeStatusMessage.failure_from_node( action, action.execution_id, result="Action not callable", started_at=action.started_at) except Exception as e: self.logger.exception( f"Failed to execute {action.label}-{action.execution_id}") action_result = NodeStatusMessage.failure_from_node( action, action.execution_id, result=repr(e), started_at=action.started_at) else: self.logger.error( f"App {self.__class__.__name__} has no method {action.name}") action_result = NodeStatusMessage.failure_from_node( action, action.execution_id, result="Action does not exist", started_at=action.started_at) await self.redis.xadd( results_stream, {action.execution_id: message_dumps(action_result)})
def control_workflow(execution): data = request.get_json() status = data['status'] workflow = workflow_getter(execution.workflow_id) # The resource factory returns the WorkflowStatus model but we want the string of the execution ID execution_id = str(execution.execution_id) # TODO: add in pause/resume here. Workers need to store and recover state for this if status == 'abort': logger.info( f"User '{get_jwt_claims().get('username', None)}' aborting workflow: {execution_id}" ) message = { "execution_id": execution_id, "status": status, "workflow": workflow_schema.dumps(workflow) } current_app.running_context.cache.smove( static.REDIS_PENDING_WORKFLOWS, static.REDIS_ABORTING_WORKFLOWS, execution_id) current_app.running_context.cache.xadd(static.REDIS_WORKFLOW_CONTROL, message) return None, HTTPStatus.NO_CONTENT elif status == 'trigger': if execution.status not in (StatusEnum.PENDING, StatusEnum.EXECUTING, StatusEnum.AWAITING_DATA): return invalid_input_problem( "workflow", "trigger", execution_id, errors=[ "Workflow must be in a running state to accept triggers." ]) trigger_id = data.get('trigger_id') if not trigger_id: return invalid_input_problem( "workflow", "trigger", execution_id, errors=["ID of the trigger must be specified in trigger_id."]) seen = False for trigger in workflow.triggers: if str(trigger.id_) == trigger_id: seen = True if not seen: return invalid_input_problem( "workflow", "trigger", execution_id, errors=[ f"trigger_id {trigger_id} was not found in this workflow." ]) trigger_stream = f"{execution_id}-{trigger_id}:triggers" try: info = current_app.running_context.cache.xinfo_stream( trigger_stream) stream_length = info["length"] except Exception: stream_length = 0 if stream_length > 0: return invalid_input_problem( "workflow", "trigger", execution_id, errors=[f"This trigger has already received data."]) trigger_data = data.get('trigger_data') logger.info( f"User '{get_jwt_claims().get('username', None)}' triggering workflow: {execution_id} at trigger " f"{trigger_id} with data {trigger_data}") current_app.running_context.cache.xadd( trigger_stream, {execution_id: message_dumps({"trigger_data": trigger_data})}) return jsonify({"trigger_stream": trigger_stream}), HTTPStatus.OK