async def execute_action(self, action: Action):
""" Execute an action and ship its result """
self.logger.debug(
f"Attempting execution of: {action.label}-{action.execution_id}")
self.console_logger.handlers[0].stream.set_channel(
f"{action.execution_id}:console")
if hasattr(self, action.name):
start_action_msg = NodeStatus.executing_from_node(
action, action.execution_id)
await self.redis.lpush(action.execution_id,
message_dumps(start_action_msg))
try:
func = getattr(self, action.name, None)
if callable(func):
if len(action.parameters) < 1:
result = await func()
else:
result = await func(
**{p.name: p.value
for p in action.parameters})
action_result = NodeStatus.success_from_node(
action, action.execution_id, result)
self.logger.debug(
f"Executed {action.label}-{action.id_} with result: {result}"
)
else:
self.logger.error(
f"App {self.__class__.__name__}.{action.name} is not callable"
)
action_result = NodeStatus.failure_from_node(
action,
action.execution_id,
error="Action not callable")
except Exception as e:
action_result = NodeStatus.failure_from_node(
action, action.execution_id, error=repr(e))
self.logger.exception(
f"Failed to execute {action.label}-{action.id_}")
await self.redis.lpush(action.execution_id,
message_dumps(action_result))
else:
self.logger.error(
f"App {self.__class__.__name__} has no method {action.name}")
action_result = NodeStatus.failure_from_node(
action, action.execution_id, error="Action does not exist")
await self.redis.lpush(action.execution_id,
message_dumps(action_result))
async def send_status_update(session, execution_id, message, headers=None):
import aiohttp
""" Forms and sends a JSONPatch message to the api_gateway to update the status of an action or workflow """
if message is None:
return None
patches = get_patches(message)
if len(patches) < 1:
raise ValueError(
f"Attempting to send improper message type: {type(message)}")
params = {"event": message.status.value}
url = f"{config.API_GATEWAY_URI}/walkoff/api/internal/workflowstatus/{execution_id}"
headers, token = await get_walkoff_auth_header(session)
headers["content-type"] = "application/json"
try:
async with session.patch(url,
data=message_dumps(patches),
params=params,
headers=headers,
timeout=5) as resp:
if resp.content_type == "application/json":
results = await resp.json()
logger.debug(f"API-Gateway status update response: {results}")
return results
except aiohttp.ClientConnectionError as e:
logger.error(f"Could not send status message to {url}: {e!r}")
except Exception as e:
logger.error(f"Unknown error while sending message to {url}: {e!r}")
async def execute_parallel_action(self, node: Action, parameters):
schedule_tasks = []
actions = set()
action_to_parallel_map = {}
results = []
parallel_parameter = [p for p in node.parameters if p.parallelized]
unparallelized = list(set(node.parameters) - set(parallel_parameter))
for i, value in enumerate(parallel_parameter[0].value):
new_value = [value]
# params = node.append(array[i])
params = []
params.extend(unparallelized)
params.append(
Parameter(parallel_parameter[0].name,
value=new_value,
variant=ParameterVariant.STATIC_VALUE))
# params.append([parameter])
act = Action(node.name,
node.position,
node.app_name,
node.app_version,
f"{node.name}:shard_{value}",
node.priority,
parameters=params,
execution_id=node.execution_id)
actions.add(act.id_)
schedule_tasks.append(
asyncio.create_task(self.schedule_node(act, {}, {})))
action_to_parallel_map[act.id_] = new_value
self.parallel_in_process[act.id_] = act
# self.in_process.pop(node.id_)
exceptions = await asyncio.gather(*schedule_tasks,
return_exceptions=True)
while not actions.intersection(set(
self.parallel_accumulator.keys())) == actions:
await asyncio.sleep(0)
for a in actions:
contents = self.parallel_accumulator[a]
for individual in contents:
results.append(individual)
self.accumulator[node.id_] = results
# self.accumulator[node.id_] = [self.parallel_accumulator[a] for a in actions]
status = NodeStatusMessage.success_from_node(
node,
self.workflow.execution_id,
self.accumulator[node.id_],
parameters=parameters,
started_at=node.started_at)
await self.redis.xadd(self.results_stream,
{status.execution_id: message_dumps(status)})
async def send_status_update(redis, execution_id, workflow_id, message):
""" Forms and sends a JSONPatch message to the api_gateway to update the status of an action or workflow """
if message is None:
return None
patches = {
"execution_id": execution_id,
"workflow_id": workflow_id,
"message": message_dumps(get_patches(message)),
"type": "workflow" if type(message) is WorkflowStatusMessage else "node"
}
# try:
logger.info(f"Sending result {patches}")
await redis.lpush(static.REDIS_RESULTS_QUEUE, json.dumps(patches))
async def test_execute_action(app, action, redis):
assert await redis.lpop(action.execution_id) is None
await app.execute_action(action)
returned = (await redis.lpop(action.execution_id)).decode("utf-8")
returned = json.loads(returned)
result = 3
action_result = NodeStatusMessage.success_from_node(action, action.execution_id, result)
to_compare = message_dumps(action_result)
to_compare = json.loads(to_compare)
for key in returned.keys():
if key != "completed_at":
assert returned[key] == to_compare[key]
async def send_message(self, message: Union[NodeStatus, WorkflowStatus]):
""" Forms and sends a JSONPatch message to the api_gateway to update the status of an action or workflow """
patches = self.get_patches(message)
if patches is None:
raise ValueError(
f"Attempting to send improper message type: {type(message)}")
data = message_dumps(patches)
params = {"event": message.status.value}
url = f"{config['WORKER']['api_gateway_uri']}/iapi/workflowstatus/{self.workflow.execution_id}"
try:
async with self.session.patch(url, data=data,
params=params) as resp:
return resp.json(loads=message_loads)
except aiohttp.ClientConnectionError as e:
logger.error(f"Could not send status message to {url}: {e!r}")
async def execute_transform(self, transform, parents):
""" Execute an transform and ship its result """
logger.debug(
f"Attempting evaluation of: {transform.label}-{self.workflow.execution_id}"
)
try:
result = transform(
parents, self.accumulator) # run transform on parent's result
status = NodeStatusMessage.success_from_node(
transform,
self.workflow.execution_id,
result,
parameters={},
started_at=transform.started_at)
logger.info(
f"Transform {transform.label}-succeeded with result: {result}")
except TransformException as e:
logger.exception(
f"Worker received error for {transform.name}-{self.workflow.execution_id}"
)
aeval = Interpreter()
aeval(transform.transform)
if len(aeval.error) > 0:
error_tuple = (aeval.error[0]).get_error()
ret = error_tuple[0] + "(): " + error_tuple[1]
status = NodeStatusMessage.failure_from_node(
transform,
self.workflow.execution_id,
result=ret,
started_at=transform.started_at,
parameters={})
except Exception as e:
logger.exception(
f"Something bad happened in Transform evaluation: {e!r}")
return
# Send the status message through redis to ensure get_action_results completes it correctly
await self.redis.xadd(self.results_stream,
{status.execution_id: message_dumps(status)})
async def control_workflow(
request: Request,
execution,
workflow_to_control: ControlWorkflow,
workflow_status_col: AsyncIOMotorCollection = Depends(get_mongo_c)):
"""
Pause, resume, or abort a workflow currently executing in WALKOFF.
"""
execution = await mongo_helpers.get_item(workflow_status_col,
WorkflowStatus,
execution,
id_key="execution_id")
walkoff_db = get_mongo_d(request)
workflow_col = walkoff_db.workflows
curr_user_id = await get_jwt_identity(request)
workflow_id = execution.workflow_id
data = dict(workflow_to_control)
status = data['status']
workflow: WorkflowModel = await mongo_helpers.get_item(
workflow_col, WorkflowModel, workflow_id)
# workflow = workflow_getter(execution.workflow_id, workflow_status_col)
# The resource factory returns the WorkflowStatus model but we want the string of the execution ID
execution_id = str(execution.execution_id)
to_execute = await auth_check(workflow,
curr_user_id,
"execute",
walkoff_db=walkoff_db)
# TODO: add in pause/resume here. Workers need to store and recover state for this
if to_execute:
if status.lower() == 'abort':
logger.info(
f"User '{(await get_jwt_claims(request)).get('username', None)}' aborting workflow: {execution_id}"
)
message = {
"execution_id": execution_id,
"status": status,
"workflow": dict(workflow)
}
async with connect_to_aioredis_pool(config.REDIS_URI) as conn:
await conn.smove(static.REDIS_PENDING_WORKFLOWS,
static.REDIS_ABORTING_WORKFLOWS, execution_id)
await conn.xadd(static.REDIS_WORKFLOW_CONTROL, message)
return None, HTTPStatus.NO_CONTENT
elif status.lower() == 'trigger':
if execution.status not in (StatusEnum.PENDING,
StatusEnum.EXECUTING,
StatusEnum.AWAITING_DATA):
raise InvalidInputException(
"workflow",
"trigger",
execution_id,
errors=[
"Workflow must be in a running state to accept triggers."
])
trigger_id = data.get('trigger_id')
if not trigger_id:
raise InvalidInputException(
"workflow",
"trigger",
execution_id,
errors=[
"ID of the trigger must be specified in trigger_id."
])
seen = False
for trigger in workflow.triggers:
if str(trigger.id_) == trigger_id:
seen = True
if not seen:
raise InvalidInputException(
"workflow",
"trigger",
execution_id,
errors=[
f"trigger_id {trigger_id} was not found in this workflow."
])
trigger_stream = f"{execution_id}-{trigger_id}:triggers"
try:
async with connect_to_aioredis_pool(config.REDIS_URI) as conn:
info = await conn.xinfo_stream(trigger_stream)
stream_length = info["length"]
except Exception:
stream_length = 0
if stream_length > 0:
return InvalidInputException(
"workflow",
"trigger",
execution_id,
errors=[f"This trigger has already received data."])
trigger_data = data.get('trigger_data')
logger.info(
f"User '{(await get_jwt_claims(request)).get('username', None)}' triggering workflow: {execution_id} at trigger "
f"{trigger_id} with data {trigger_data}")
async with connect_to_aioredis_pool(config.REDIS_URI) as conn:
await conn.xadd(trigger_stream, {
execution_id:
message_dumps({"trigger_data": trigger_data})
})
return ({"trigger_stream": trigger_stream})
else:
return None
async def evaluate_condition(self, condition, parents, children):
"""
TODO: This will change when we implement a better UI element for it. For now, if an action is given a user
defined name like "Hello World", it would be referenced by the variable name "Hello_World" in the
conditional script. All whitespace in the action name is replaced by '_'. This is clearly problematic
if a user has an action named "Hello World" as well as "Hello_World". In this case, we cannot be sure
which is being referenced in the conditional and must raise an exception.
"""
logger.debug(
f"Attempting evaluation of: {condition.label}-{self.workflow.execution_id}"
)
try:
child_id = condition(parents, children, self.accumulator)
selected_node = children.pop(child_id)
status = NodeStatusMessage.success_from_node(
condition,
self.workflow.execution_id,
selected_node.name,
parameters={},
started_at=condition.started_at)
logger.info(
f"Condition selected node: {selected_node.label}-{self.workflow.execution_id}"
)
# We preemptively schedule all branches of execution so we must cancel all "false" branches here
for child in children.values():
if self.parent_map[child.id_] == 1:
await self.cancel_subgraph(child)
except ConditionException as e:
logger.exception(
f"Worker received error for {condition.name}-{self.workflow.execution_id}"
)
aeval = Interpreter()
aeval(condition.conditional)
if len(aeval.error) > 0:
error_tuple = (aeval.error[0]).get_error()
ret = error_tuple[0] + "(): " + error_tuple[1]
status = NodeStatusMessage.failure_from_node(
condition,
self.workflow.execution_id,
result=ret,
parameters={},
started_at=condition.started_at)
except KeyError as e:
logger.exception(
f"Worker received error for {condition.name}-{self.workflow.execution_id}"
)
status = NodeStatusMessage.failure_from_node(
condition,
self.workflow.execution_id,
result=
"ConditionError(): ensure that a non-parent node is selected and that the node name is not a string.",
parameters={},
started_at=condition.started_at)
except Exception as e:
logger.exception(
f"Something bad happened in Condition evaluation: {e!r}")
return
# Send the status message through redis to ensure get_action_results completes it correctly
await self.redis.xadd(self.results_stream,
{status.execution_id: message_dumps(status)})
async def execute_action(self, action: Action):
""" Execute an action, and push its result to Redis. """
# TODO: Is there a better way to do this?
self.logger.handlers[0].stream.execution_id = action.execution_id
self.logger.handlers[0].stream.workflow_id = action.workflow_id
self.logger.debug(
f"Attempting execution of: {action.label}-{action.execution_id}")
self.current_execution_id = action.execution_id
self.current_workflow_id = action.workflow_id
results_stream = f"{action.execution_id}:results"
if hasattr(self, action.name):
# Tell everyone we started execution
action.started_at = datetime.datetime.now()
start_action_msg = NodeStatusMessage.executing_from_node(
action, action.execution_id, started_at=action.started_at)
await self.redis.xadd(
results_stream,
{action.execution_id: message_dumps(start_action_msg)})
try:
func = getattr(self, action.name, None)
if callable(func):
if len(action.parameters) < 1:
result = await func()
else:
params = {}
for p in action.parameters:
if p.variant == ParameterVariant.GLOBAL:
key = config.get_from_file(
config.ENCRYPTION_KEY_PATH, 'rb')
params[p.name] = fernet_decrypt(key, p.value)
else:
params[p.name] = p.value
result = await func(**params)
action_result = NodeStatusMessage.success_from_node(
action,
action.execution_id,
result=result,
started_at=action.started_at)
self.logger.debug(
f"Executed {action.label}-{action.execution_id} "
f"with result: {result}")
else:
self.logger.error(
f"App {self.__class__.__name__}.{action.name} is not callable"
)
action_result = NodeStatusMessage.failure_from_node(
action,
action.execution_id,
result="Action not callable",
started_at=action.started_at)
except Exception as e:
self.logger.exception(
f"Failed to execute {action.label}-{action.execution_id}")
action_result = NodeStatusMessage.failure_from_node(
action,
action.execution_id,
result=repr(e),
started_at=action.started_at)
else:
self.logger.error(
f"App {self.__class__.__name__} has no method {action.name}")
action_result = NodeStatusMessage.failure_from_node(
action,
action.execution_id,
result="Action does not exist",
started_at=action.started_at)
await self.redis.xadd(
results_stream,
{action.execution_id: message_dumps(action_result)})
def control_workflow(execution):
data = request.get_json()
status = data['status']
workflow = workflow_getter(execution.workflow_id)
# The resource factory returns the WorkflowStatus model but we want the string of the execution ID
execution_id = str(execution.execution_id)
# TODO: add in pause/resume here. Workers need to store and recover state for this
if status == 'abort':
logger.info(
f"User '{get_jwt_claims().get('username', None)}' aborting workflow: {execution_id}"
)
message = {
"execution_id": execution_id,
"status": status,
"workflow": workflow_schema.dumps(workflow)
}
current_app.running_context.cache.smove(
static.REDIS_PENDING_WORKFLOWS, static.REDIS_ABORTING_WORKFLOWS,
execution_id)
current_app.running_context.cache.xadd(static.REDIS_WORKFLOW_CONTROL,
message)
return None, HTTPStatus.NO_CONTENT
elif status == 'trigger':
if execution.status not in (StatusEnum.PENDING, StatusEnum.EXECUTING,
StatusEnum.AWAITING_DATA):
return invalid_input_problem(
"workflow",
"trigger",
execution_id,
errors=[
"Workflow must be in a running state to accept triggers."
])
trigger_id = data.get('trigger_id')
if not trigger_id:
return invalid_input_problem(
"workflow",
"trigger",
execution_id,
errors=["ID of the trigger must be specified in trigger_id."])
seen = False
for trigger in workflow.triggers:
if str(trigger.id_) == trigger_id:
seen = True
if not seen:
return invalid_input_problem(
"workflow",
"trigger",
execution_id,
errors=[
f"trigger_id {trigger_id} was not found in this workflow."
])
trigger_stream = f"{execution_id}-{trigger_id}:triggers"
try:
info = current_app.running_context.cache.xinfo_stream(
trigger_stream)
stream_length = info["length"]
except Exception:
stream_length = 0
if stream_length > 0:
return invalid_input_problem(
"workflow",
"trigger",
execution_id,
errors=[f"This trigger has already received data."])
trigger_data = data.get('trigger_data')
logger.info(
f"User '{get_jwt_claims().get('username', None)}' triggering workflow: {execution_id} at trigger "
f"{trigger_id} with data {trigger_data}")
current_app.running_context.cache.xadd(
trigger_stream,
{execution_id: message_dumps({"trigger_data": trigger_data})})
return jsonify({"trigger_stream": trigger_stream}), HTTPStatus.OK
