def initialize_threading(self, app, pids=None):
"""Initialize the multiprocessing communication threads, allowing for parallel execution of workflows.
Args:
app (FlaskApp): The current_app object
pids (list[Process], optional): Optional list of spawned processes. Defaults to None
"""
if not (os.path.exists(walkoff.config.Config.ZMQ_PUBLIC_KEYS_PATH) and
os.path.exists(walkoff.config.Config.ZMQ_PRIVATE_KEYS_PATH)):
logging.fatal(
"Certificates are missing - run generate_certificates.py script first."
)
sys.exit(0)
self.pids = pids
self.ctx = zmq.Context.instance()
self.auth = ThreadAuthenticator()
self.auth.start()
self.auth.allow('127.0.0.1')
self.auth.configure_curve(
domain='*', location=walkoff.config.Config.ZMQ_PUBLIC_KEYS_PATH)
self.manager = WorkflowExecutionController(self.cache)
self.receiver = Receiver(app)
self.receiver_thread = threading.Thread(
target=self.receiver.receive_results)
self.receiver_thread.start()
self.threading_is_initialized = True
logger.debug('Controller threading initialized')
def initialize_threading(self, pids=None):
"""Initialize the multiprocessing communication threads, allowing for parallel execution of workflows.
"""
if not (os.path.exists(walkoff.config.paths.zmq_public_keys_path) and
os.path.exists(walkoff.config.paths.zmq_private_keys_path)):
logging.error(
"Certificates are missing - run generate_certificates.py script first."
)
sys.exit(0)
self.pids = pids
self.ctx = zmq.Context.instance()
self.auth = ThreadAuthenticator(self.ctx)
self.auth.start()
self.auth.allow('127.0.0.1')
self.auth.configure_curve(
domain='*', location=walkoff.config.paths.zmq_public_keys_path)
self.manager = LoadBalancer(self.ctx)
self.receiver = Receiver(self.ctx)
self.receiver_thread = threading.Thread(
target=self.receiver.receive_results)
self.receiver_thread.start()
self.manager_thread = threading.Thread(
target=self.manager.manage_workflows)
self.manager_thread.start()
self.threading_is_initialized = True
logger.debug('Controller threading initialized')
def initialize_threading(self, app, pids=None):
"""Initialize the multiprocessing communication threads, allowing for parallel execution of workflows.
Args:
app (FlaskApp): The current_app object
pids (list[Process], optional): Optional list of spawned processes. Defaults to None
"""
if walkoff.config.Config.SEPARATE_RECEIVER:
pass
self.pids = pids
if 'zmq' in [
walkoff.config.Config.WORKFLOW_COMMUNICATION_HANDLER,
walkoff.config.Config.WORKFLOW_RESULTS_HANDLER
]:
# Only run the threadauthenticator if ZMQ is in use
self.ctx = zmq.Context.instance()
self.auth = ThreadAuthenticator()
self.auth.start()
# TODO: self.auth.allow('127.0.0.1')
self.auth.configure_curve(
domain='*',
location=walkoff.config.Config.ZMQ_PUBLIC_KEYS_PATH)
self.zmq_workflow_comm = make_communication_sender()
with app.app_context():
data = {'execution_db': current_app.running_context.execution_db}
self.results_sender = make_results_sender(**data)
if not walkoff.config.Config.SEPARATE_RECEIVER:
data = {'current_app': app}
self.receiver = make_results_receiver(**data)
self.receiver_thread = threading.Thread(
target=self.receiver.receive_results)
self.receiver_thread.start()
self.threading_is_initialized = True
logger.debug('Controller threading initialized')
def initialize_threading(self, app, pids=None):
"""Initialize the multiprocessing communication threads, allowing for parallel execution of workflows.
Args:
app (FlaskApp): The current_app object
pids (list[Process], optional): Optional list of spawned processes. Defaults to None
"""
if walkoff.config.Config.SEPARATE_RECEIVER:
pass
self.pids = pids
if 'zmq' in [walkoff.config.Config.WORKFLOW_COMMUNICATION_HANDLER,
walkoff.config.Config.WORKFLOW_RESULTS_HANDLER]:
# Only run the threadauthenticator if ZMQ is in use
self.ctx = zmq.Context.instance()
self.auth = ThreadAuthenticator()
self.auth.start()
# TODO: self.auth.allow('127.0.0.1')
self.auth.configure_curve(domain='*', location=walkoff.config.Config.ZMQ_PUBLIC_KEYS_PATH)
self.zmq_workflow_comm = make_communication_sender()
with app.app_context():
data = {'execution_db': current_app.running_context.execution_db}
self.results_sender = make_results_sender(**data)
if not walkoff.config.Config.SEPARATE_RECEIVER:
data = {'current_app': app}
self.receiver = make_results_receiver(**data)
self.receiver_thread = threading.Thread(target=self.receiver.receive_results)
self.receiver_thread.start()
self.threading_is_initialized = True
logger.debug('Controller threading initialized')
class MultiprocessedExecutor(object):
def __init__(self, cache, config):
"""Initializes a multiprocessed executor, which will handle the execution of workflows.
"""
self.threading_is_initialized = False
self.id = "controller"
self.pids = None
self.workflows_executed = 0
self.ctx = None # TODO: Test if you can always use the singleton
self.auth = None
self.zmq_workflow_comm = None
self.receiver = None
self.receiver_thread = None
self.cache = cache
self.config = config
self.execution_db = ExecutionDatabase.instance
self.results_sender = None
key = PrivateKey(
walkoff.config.Config.
SERVER_PRIVATE_KEY[:nacl.bindings.crypto_box_SECRETKEYBYTES])
worker_key = PrivateKey(
walkoff.config.Config.CLIENT_PRIVATE_KEY[:nacl.bindings.
crypto_box_SECRETKEYBYTES]
).public_key
self.__box = Box(key, worker_key)
def initialize_threading(self, app, pids=None):
"""Initialize the multiprocessing communication threads, allowing for parallel execution of workflows.
Args:
app (FlaskApp): The current_app object
pids (list[Process], optional): Optional list of spawned processes. Defaults to None
"""
if walkoff.config.Config.SEPARATE_RECEIVER:
pass
self.pids = pids
if 'zmq' in [
walkoff.config.Config.WORKFLOW_COMMUNICATION_HANDLER,
walkoff.config.Config.WORKFLOW_RESULTS_HANDLER
]:
# Only run the threadauthenticator if ZMQ is in use
self.ctx = zmq.Context.instance()
self.auth = ThreadAuthenticator()
self.auth.start()
# TODO: self.auth.allow('127.0.0.1')
self.auth.configure_curve(
domain='*',
location=walkoff.config.Config.ZMQ_PUBLIC_KEYS_PATH)
self.zmq_workflow_comm = make_communication_sender()
with app.app_context():
data = {'execution_db': current_app.running_context.execution_db}
self.results_sender = make_results_sender(**data)
if not walkoff.config.Config.SEPARATE_RECEIVER:
data = {'current_app': app}
self.receiver = make_results_receiver(**data)
self.receiver_thread = threading.Thread(
target=self.receiver.receive_results)
self.receiver_thread.start()
self.threading_is_initialized = True
logger.debug('Controller threading initialized')
def wait_and_reset(self, num_workflows):
"""Waits for all of the workflows to be completed
Args:
num_workflows (int): The number of workflows to wait for
"""
timeout = 0
shutdown = 10
while timeout < shutdown:
if self.receiver is not None and num_workflows == self.receiver.workflows_executed:
break
timeout += 0.1
gevent.sleep(0.1)
assert (num_workflows == self.receiver.workflows_executed)
self.receiver.workflows_executed = 0
def shutdown_pool(self):
"""Shuts down the threadpool"""
if self.zmq_workflow_comm:
self.zmq_workflow_comm.send_exit_to_workers()
if not walkoff.config.Config.SEPARATE_WORKERS:
shutdown_procs(self.pids)
if self.receiver_thread:
self.receiver.thread_exit = True
self.receiver_thread.join(timeout=1)
self.threading_is_initialized = False
logger.debug('Controller thread pool shutdown')
if self.auth:
self.auth.stop()
if self.ctx:
self.ctx.destroy()
self.cleanup_threading()
self.execution_db.tear_down()
return
def cleanup_threading(self):
"""Once the threadpool has been shutdown, clear out all of the data structures used in the pool"""
self.pids = []
self.receiver_thread = None
self.workflows_executed = 0
self.threading_is_initialized = False
self.zmq_workflow_comm = None
self.receiver = None
def execute_workflow(self,
workflow_id,
execution_id_in=None,
start=None,
start_arguments=None,
resume=False,
environment_variables=None,
user=None):
"""Executes a workflow
Args:
workflow_id (Workflow): The Workflow to be executed.
execution_id_in (UUID, optional): The optional execution ID to provide for the workflow. Should only be
used (and is required) when resuming a workflow. Must be valid UUID4. Defaults to None.
start (UUID, optional): The ID of the first, or starting action. Defaults to None.
start_arguments (list[Argument]): The arguments to the starting action of the workflow. Defaults to None.
resume (bool, optional): Optional boolean to resume a previously paused workflow. Defaults to False.
environment_variables (list[EnvironmentVariable]): Optional list of environment variables to pass into
the workflow. These will not be persistent.
user (str, Optional): The username of the user who requested that this workflow be executed. Defaults
to None.
Returns:
(UUID): The execution ID of the Workflow.
"""
workflow = self.execution_db.session.query(Workflow).filter_by(
id=workflow_id).first()
if not workflow:
logger.error(
'Attempted to execute workflow {} which does not exist'.format(
execution_id_in))
return None, 'Attempted to execute workflow which does not exist'
execution_id = execution_id_in if execution_id_in else str(
uuid.uuid4())
if start is not None:
logger.info(
'User {0} executing workflow {1} (id={2}) with starting action {3}'
.format(user, workflow.name, workflow.id, start))
else:
logger.info(
'User {0} executing workflow {1} (id={2}) with default starting action {3}'
.format(user, workflow.name, workflow.id, workflow.start))
workflow_data = {
'execution_id': execution_id,
'id': str(workflow.id),
'name': workflow.name
}
data = {}
if user:
data['user'] = user
self._log_and_send_event(WalkoffEvent.WorkflowExecutionPending,
sender=workflow_data,
workflow=workflow,
data=data)
self.__add_workflow_to_queue(workflow.id, execution_id, start,
start_arguments, resume,
environment_variables, user)
self._log_and_send_event(WalkoffEvent.SchedulerJobExecuted, data=data)
return execution_id
def __add_workflow_to_queue(self,
workflow_id,
workflow_execution_id,
start=None,
start_arguments=None,
resume=False,
environment_variables=None,
user=None):
message = self.results_sender.create_workflow_request_message(
workflow_id, workflow_execution_id, start, start_arguments, resume,
environment_variables, user)
self.cache.lpush("request_queue", self.__box.encrypt(message))
def pause_workflow(self, execution_id, user=None):
"""Pauses a workflow that is currently executing.
Args:
execution_id (UUID): The execution id of the workflow.
Returns:
(bool): True if Workflow successfully paused, False otherwise
"""
logger.info('User {0} pausing workflow {1}'.format(user, execution_id))
workflow_status = self.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status and workflow_status.status == WorkflowStatusEnum.running:
self.zmq_workflow_comm.pause_workflow(execution_id)
return True
else:
logger.warning(
'Cannot pause workflow {0}. Invalid key, or workflow not running.'
.format(execution_id))
return False
def resume_workflow(self, execution_id, user=None):
"""Resumes a workflow that is currently paused.
Args:
execution_id (UUID): The execution id of the workflow.
user (str, Optional): The username of the user who requested that this workflow be resumed. Defaults
to None.
Returns:
(bool): True if workflow successfully resumed, False otherwise
"""
logger.info('User {0} resuming workflow {1}'.format(
user, execution_id))
workflow_status = self.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status and workflow_status.status == WorkflowStatusEnum.paused:
saved_state = self.execution_db.session.query(
SavedWorkflow).filter_by(
workflow_execution_id=execution_id).first()
workflow = self.execution_db.session.query(Workflow).filter_by(
id=workflow_status.workflow_id).first()
data = {"execution_id": execution_id}
if user:
data['user'] = user
self._log_and_send_event(WalkoffEvent.WorkflowResumed,
sender=workflow,
data=data)
start = saved_state.action_id if saved_state else workflow.start
self.execute_workflow(workflow.id,
execution_id_in=execution_id,
start=start,
resume=True,
user=user)
return True
else:
logger.warning(
'Cannot resume workflow {0}. Invalid key, or workflow not paused.'
.format(execution_id))
return False
def abort_workflow(self, execution_id, user=None):
"""Abort a workflow
Args:
execution_id (UUID): The execution id of the workflow.
user (str, Optional): The username of the user who requested that this workflow be aborted. Defaults
to None.
Returns:
(bool): True if successfully aborted workflow, False otherwise
"""
logger.info('User {0} aborting workflow {1}'.format(
user, execution_id))
workflow_status = self.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status:
if workflow_status.status in [
WorkflowStatusEnum.pending, WorkflowStatusEnum.paused,
WorkflowStatusEnum.awaiting_data
]:
workflow = self.execution_db.session.query(Workflow).filter_by(
id=workflow_status.workflow_id).first()
if workflow is not None:
data = {}
if user:
data['user'] = user
self._log_and_send_event(WalkoffEvent.WorkflowAborted,
sender={
'execution_id': execution_id,
'id':
workflow_status.workflow_id,
'name': workflow.name
},
workflow=workflow,
data=data)
elif workflow_status.status == WorkflowStatusEnum.running:
self.zmq_workflow_comm.abort_workflow(execution_id)
return True
else:
logger.warning(
'Cannot resume workflow {0}. Invalid key, or workflow already shutdown.'
.format(execution_id))
return False
def resume_trigger_step(self,
execution_id,
data_in,
arguments=None,
user=None):
"""Resumes a workflow awaiting trigger data, if the conditions are met.
Args:
execution_id (UUID): The execution ID of the workflow
data_in (dict): The data to send to the trigger
arguments (list[Argument], optional): Optional list of new Arguments for the trigger action.
Defaults to None.
user (str, Optional): The username of the user who requested that this workflow be resumed. Defaults
to None.
Returns:
(bool): True if successfully resumed trigger step, false otherwise
"""
logger.info('User {0} resuming workflow {1} from trigger'.format(
user, execution_id))
saved_state = self.execution_db.session.query(SavedWorkflow).filter_by(
workflow_execution_id=execution_id).first()
workflow = self.execution_db.session.query(Workflow).filter_by(
id=saved_state.workflow_id).first()
action_execution_strategy = make_execution_strategy(
self.config,
RestrictedWorkflowContext.from_workflow(workflow, execution_id))
accumulator = make_accumulator(execution_id)
executed = False
exec_action = None
for action in workflow.actions:
if action.id == saved_state.action_id:
exec_action = action
executed = action.execute_trigger(action_execution_strategy,
data_in, accumulator)
break
data = {'workflow_execution_id': execution_id}
if user:
data['user'] = user
if executed:
self._log_and_send_event(
WalkoffEvent.TriggerActionTaken,
sender=exec_action,
data={'workflow_execution_id': execution_id},
workflow=workflow)
self.execute_workflow(workflow.id,
execution_id_in=execution_id,
start=str(saved_state.action_id),
start_arguments=arguments,
resume=True,
user=user)
return True
else:
self._log_and_send_event(
WalkoffEvent.TriggerActionNotTaken,
sender=exec_action,
data={'workflow_execution_id': execution_id},
workflow=workflow)
return False
def get_waiting_workflows(self):
"""Gets a list of the execution IDs of workflows currently awaiting data to be sent to a trigger.
Returns:
(list[UUID]): A list of execution IDs of workflows currently awaiting data to be sent to a trigger.
"""
self.execution_db.session.expire_all()
wf_statuses = self.execution_db.session.query(
WorkflowStatus).filter_by(
status=WorkflowStatusEnum.awaiting_data).all()
return [str(wf_status.execution_id) for wf_status in wf_statuses]
def get_workflow_status(self, execution_id):
"""Gets the current status of a workflow by its execution ID
Args:
execution_id (UUID): The execution ID of the workflow
Returns:
(int): The status of the workflow
"""
workflow_status = self.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status:
return workflow_status.status
else:
logger.error(
"Workflow execution id {} does not exist in WorkflowStatus table."
).format(execution_id)
return 0
def _log_and_send_event(self,
event,
sender=None,
data=None,
workflow=None):
sender = sender or self
self.results_sender.handle_event(workflow,
sender,
event=event,
data=data)
class MultiprocessedExecutor(object):
def __init__(self, cache, event_logger):
"""Initializes a multiprocessed executor, which will handle the execution of workflows.
"""
self.threading_is_initialized = False
self.id = "controller"
self.pids = None
self.workflows_executed = 0
self.ctx = None # TODO: Test if you can always use the singleton
self.auth = None
self.manager = None
self.receiver = None
self.receiver_thread = None
self.cache = cache
self.event_logger = event_logger
self.execution_db = ExecutionDatabase.instance
def initialize_threading(self, app, pids=None):
"""Initialize the multiprocessing communication threads, allowing for parallel execution of workflows.
Args:
app (FlaskApp): The current_app object
pids (list[Process], optional): Optional list of spawned processes. Defaults to None
"""
if not (os.path.exists(walkoff.config.Config.ZMQ_PUBLIC_KEYS_PATH) and
os.path.exists(walkoff.config.Config.ZMQ_PRIVATE_KEYS_PATH)):
logging.fatal(
"Certificates are missing - run generate_certificates.py script first."
)
sys.exit(0)
self.pids = pids
self.ctx = zmq.Context.instance()
self.auth = ThreadAuthenticator()
self.auth.start()
self.auth.allow('127.0.0.1')
self.auth.configure_curve(
domain='*', location=walkoff.config.Config.ZMQ_PUBLIC_KEYS_PATH)
self.manager = WorkflowExecutionController(self.cache)
self.receiver = Receiver(app)
self.receiver_thread = threading.Thread(
target=self.receiver.receive_results)
self.receiver_thread.start()
self.threading_is_initialized = True
logger.debug('Controller threading initialized')
def wait_and_reset(self, num_workflows):
"""Waits for all of the workflows to be completed
Args:
num_workflows (int): The number of workflows to wait for
"""
timeout = 0
shutdown = 10
while timeout < shutdown:
if self.receiver is not None and num_workflows == self.receiver.workflows_executed:
break
timeout += 0.1
gevent.sleep(0.1)
assert (num_workflows == self.receiver.workflows_executed)
self.receiver.workflows_executed = 0
def shutdown_pool(self):
"""Shuts down the threadpool"""
self.manager.send_exit_to_worker_comms()
if len(self.pids) > 0:
for p in self.pids:
if p.is_alive():
logger.info(
'Multiprocessed executor shutting down process {}'.
format(p))
os.kill(p.pid, signal.SIGABRT)
p.join(timeout=3)
try:
os.kill(p.pid, signal.SIGKILL)
except (OSError, AttributeError):
pass
if self.receiver_thread:
self.receiver.thread_exit = True
self.receiver_thread.join(timeout=1)
self.threading_is_initialized = False
logger.debug('Controller thread pool shutdown')
if self.auth:
self.auth.stop()
if self.ctx:
self.ctx.destroy()
self.cleanup_threading()
return
def cleanup_threading(self):
"""Once the threadpool has been shutdown, clear out all of the data structures used in the pool"""
self.pids = []
self.receiver_thread = None
self.workflows_executed = 0
self.threading_is_initialized = False
self.manager = None
self.receiver = None
def execute_workflow(self,
workflow_id,
execution_id_in=None,
start=None,
start_arguments=None,
resume=False,
environment_variables=None):
"""Executes a workflow
Args:
workflow_id (Workflow): The Workflow to be executed.
execution_id_in (UUID, optional): The optional execution ID to provide for the workflow. Should only be
used (and is required) when resuming a workflow. Must be valid UUID4. Defaults to None.
start (UUID, optional): The ID of the first, or starting action. Defaults to None.
start_arguments (list[Argument]): The arguments to the starting action of the workflow. Defaults to None.
resume (bool, optional): Optional boolean to resume a previously paused workflow. Defaults to False.
environment_variables (list[EnvironmentVariable]): Optional list of environment variables to pass into
the workflow. These will not be persistent.
Returns:
(UUID): The execution ID of the Workflow.
"""
workflow = self.execution_db.session.query(Workflow).filter_by(
id=workflow_id).first()
if not workflow:
logger.error(
'Attempted to execute workflow {} which does not exist'.format(
execution_id_in))
return None, 'Attempted to execute workflow which does not exist'
execution_id = execution_id_in if execution_id_in else str(
uuid.uuid4())
if start is not None:
logger.info(
'Executing workflow {0} (id={1}) with starting action action {2}'
.format(workflow.name, workflow.id, start))
else:
logger.info(
'Executing workflow {0} (id={1}) with default starting action'.
format(workflow.name, workflow.id, start))
workflow_data = {
'execution_id': execution_id,
'id': str(workflow.id),
'name': workflow.name
}
self._log_and_send_event(WalkoffEvent.WorkflowExecutionPending,
sender=workflow_data)
self.manager.add_workflow(workflow.id, execution_id, start,
start_arguments, resume,
environment_variables)
self._log_and_send_event(WalkoffEvent.SchedulerJobExecuted)
return execution_id
def pause_workflow(self, execution_id):
"""Pauses a workflow that is currently executing.
Args:
execution_id (UUID): The execution id of the workflow.
Returns:
(bool): True if Workflow successfully paused, False otherwise
"""
logger.info('Pausing workflow {}'.format(execution_id))
workflow_status = self.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status and workflow_status.status == WorkflowStatusEnum.running:
self.manager.pause_workflow(execution_id)
return True
else:
logger.warning(
'Cannot pause workflow {0}. Invalid key, or workflow not running.'
.format(execution_id))
return False
def resume_workflow(self, execution_id):
"""Resumes a workflow that is currently paused.
Args:
execution_id (UUID): The execution id of the workflow.
Returns:
(bool): True if workflow successfully resumed, False otherwise
"""
logger.info('Resuming workflow {}'.format(execution_id))
workflow_status = self.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status and workflow_status.status == WorkflowStatusEnum.paused:
saved_state = self.execution_db.session.query(
SavedWorkflow).filter_by(
workflow_execution_id=execution_id).first()
workflow = self.execution_db.session.query(Workflow).filter_by(
id=workflow_status.workflow_id).first()
workflow._execution_id = execution_id
self._log_and_send_event(WalkoffEvent.WorkflowResumed,
sender=workflow)
start = saved_state.action_id if saved_state else workflow.start
self.execute_workflow(workflow.id,
execution_id_in=execution_id,
start=start,
resume=True)
return True
else:
logger.warning(
'Cannot resume workflow {0}. Invalid key, or workflow not paused.'
.format(execution_id))
return False
def abort_workflow(self, execution_id):
"""Abort a workflow
Args:
execution_id (UUID): The execution id of the workflow.
Returns:
(bool): True if successfully aborted workflow, False otherwise
"""
logger.info('Aborting workflow {}'.format(execution_id))
workflow_status = self.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status:
if workflow_status.status in [
WorkflowStatusEnum.pending, WorkflowStatusEnum.paused,
WorkflowStatusEnum.awaiting_data
]:
workflow = self.execution_db.session.query(Workflow).filter_by(
id=workflow_status.workflow_id).first()
if workflow is not None:
self._log_and_send_event(WalkoffEvent.WorkflowAborted,
sender={
'execution_id': execution_id,
'id':
workflow_status.workflow_id,
'name': workflow.name
})
elif workflow_status.status == WorkflowStatusEnum.running:
self.manager.abort_workflow(execution_id)
return True
else:
logger.warning(
'Cannot resume workflow {0}. Invalid key, or workflow already shutdown.'
.format(execution_id))
return False
def resume_trigger_step(self, execution_id, data_in, arguments=None):
"""Resumes a workflow awaiting trigger data, if the conditions are met.
Args:
execution_id (UUID): The execution ID of the workflow
data_in (dict): The data to send to the trigger
arguments (list[Argument], optional): Optional list of new Arguments for the trigger action.
Defaults to None.
Returns:
(bool): True if successfully resumed trigger step, false otherwise
"""
logger.info('Resuming workflow {} from trigger'.format(execution_id))
saved_state = self.execution_db.session.query(SavedWorkflow).filter_by(
workflow_execution_id=execution_id).first()
workflow = self.execution_db.session.query(Workflow).filter_by(
id=saved_state.workflow_id).first()
workflow._execution_id = execution_id
executed = False
exec_action = None
for action in workflow.actions:
if action.id == saved_state.action_id:
exec_action = action
executed = action.execute_trigger(data_in,
saved_state.accumulator)
break
if executed:
self._log_and_send_event(
WalkoffEvent.TriggerActionTaken,
sender=exec_action,
data={'workflow_execution_id': execution_id})
self.execute_workflow(workflow.id,
execution_id_in=execution_id,
start=str(saved_state.action_id),
start_arguments=arguments,
resume=True)
return True
else:
self._log_and_send_event(
WalkoffEvent.TriggerActionNotTaken,
sender=exec_action,
data={'workflow_execution_id': execution_id})
return False
def get_waiting_workflows(self):
"""Gets a list of the execution IDs of workflows currently awaiting data to be sent to a trigger.
Returns:
(list[UUID]): A list of execution IDs of workflows currently awaiting data to be sent to a trigger.
"""
self.execution_db.session.expire_all()
wf_statuses = self.execution_db.session.query(
WorkflowStatus).filter_by(
status=WorkflowStatusEnum.awaiting_data).all()
return [str(wf_status.execution_id) for wf_status in wf_statuses]
def get_workflow_status(self, execution_id):
"""Gets the current status of a workflow by its execution ID
Args:
execution_id (UUID): The execution ID of the workflow
Returns:
(int): The status of the workflow
"""
workflow_status = self.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status:
return workflow_status.status
else:
logger.error(
"Workflow execution id {} does not exist in WorkflowStatus table."
).format(execution_id)
return 0
def _log_and_send_event(self, event, sender=None, data=None):
sender = sender or self
sender_id = sender.id if not isinstance(sender, dict) else sender['id']
self.event_logger.log(event, sender_id, data=data)
event.send(sender, data=data)
def create_case(self, case_id, subscriptions):
"""Creates a Case
Args:
case_id (int): The ID of the Case
subscriptions (list[Subscription]): List of Subscriptions to subscribe to
"""
self.manager.create_case(case_id, subscriptions)
def update_case(self, case_id, subscriptions):
"""Updates a Case
Args:
case_id (int): The ID of the Case
subscriptions (list[Subscription]): List of Subscriptions to subscribe to
"""
self.manager.create_case(case_id, subscriptions)
def delete_case(self, case_id):
"""Deletes a Case
Args:
case_id (int): The ID of the Case to delete
"""
self.manager.delete_case(case_id)
class MultiprocessedExecutor(object):
def __init__(self):
"""Initializes a multiprocessed executor, which will handle the execution of workflows.
"""
self.threading_is_initialized = False
self.id = "controller"
self.pids = None
self.workflows_executed = 0
self.ctx = None
self.auth = None
self.manager = None
self.manager_thread = None
self.receiver = None
self.receiver_thread = None
def initialize_threading(self, pids=None):
"""Initialize the multiprocessing communication threads, allowing for parallel execution of workflows.
"""
if not (os.path.exists(walkoff.config.paths.zmq_public_keys_path) and
os.path.exists(walkoff.config.paths.zmq_private_keys_path)):
logging.error(
"Certificates are missing - run generate_certificates.py script first."
)
sys.exit(0)
self.pids = pids
self.ctx = zmq.Context.instance()
self.auth = ThreadAuthenticator(self.ctx)
self.auth.start()
self.auth.allow('127.0.0.1')
self.auth.configure_curve(
domain='*', location=walkoff.config.paths.zmq_public_keys_path)
self.manager = LoadBalancer(self.ctx)
self.receiver = Receiver(self.ctx)
self.receiver_thread = threading.Thread(
target=self.receiver.receive_results)
self.receiver_thread.start()
self.manager_thread = threading.Thread(
target=self.manager.manage_workflows)
self.manager_thread.start()
self.threading_is_initialized = True
logger.debug('Controller threading initialized')
def wait_and_reset(self, num_workflows):
timeout = 0
shutdown = 10
while timeout < shutdown:
if self.receiver is not None and num_workflows == self.receiver.workflows_executed:
break
timeout += 0.1
gevent.sleep(0.1)
self.receiver.workflows_executed = 0
def shutdown_pool(self):
"""Shuts down the threadpool.
"""
self.manager.send_exit_to_worker_comms()
if self.manager_thread:
self.manager.thread_exit = True
self.manager_thread.join(timeout=1)
if len(self.pids) > 0:
for p in self.pids:
if p.is_alive():
os.kill(p.pid, signal.SIGABRT)
p.join(timeout=3)
try:
os.kill(p.pid, signal.SIGKILL)
except (OSError, AttributeError):
pass
if self.receiver_thread:
self.receiver.thread_exit = True
self.receiver_thread.join(timeout=1)
self.threading_is_initialized = False
logger.debug('Controller thread pool shutdown')
if self.auth:
self.auth.stop()
if self.ctx:
self.ctx.destroy()
self.cleanup_threading()
return
def cleanup_threading(self):
"""Once the threadpool has been shutdown, clear out all of the data structures used in the pool.
"""
self.pids = []
self.receiver_thread = None
self.manager_thread = None
self.workflows_executed = 0
self.threading_is_initialized = False
self.manager = None
self.receiver = None
def execute_workflow(self,
workflow_id,
execution_id_in=None,
start=None,
start_arguments=None,
resume=False):
"""Executes a workflow.
Args:
workflow_id (Workflow): The Workflow to be executed.
execution_id_in (str, optional): The optional execution ID to provide for the workflow. Should only be
used (and is required) when resuming a workflow. Must be valid UUID4. Defaults to None.
start (str, optional): The ID of the first, or starting action. Defaults to None.
start_arguments (list[Argument]): The arguments to the starting action of the workflow. Defaults to None.
resume (bool, optional): Optional boolean to resume a previously paused workflow. Defaults to False.
Returns:
The execution ID of the Workflow.
"""
workflow = executiondb.execution_db.session.query(Workflow).filter_by(
id=workflow_id).first()
if not workflow:
logger.error('Attempted to execute workflow which does not exist')
return None, 'Attempted to execute workflow which does not exist'
execution_id = execution_id_in if execution_id_in else str(
uuid.uuid4())
if start is not None:
logger.info('Executing workflow {0} for action {1}'.format(
workflow.name, start))
else:
logger.info(
'Executing workflow {0} with default starting action'.format(
workflow.name, start))
workflow_data = {
'execution_id': execution_id,
'id': workflow.id,
'name': workflow.name
}
WalkoffEvent.WorkflowExecutionPending.send(workflow_data)
self.manager.add_workflow(workflow.id, execution_id, start,
start_arguments, resume)
WalkoffEvent.SchedulerJobExecuted.send(self)
return execution_id
def pause_workflow(self, execution_id):
"""Pauses a workflow that is currently executing.
Args:
execution_id (str): The execution id of the workflow.
"""
workflow_status = executiondb.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status and workflow_status.status == WorkflowStatusEnum.running:
self.manager.pause_workflow(execution_id)
return True
else:
logger.warning(
'Cannot pause workflow {0}. Invalid key, or workflow not running.'
.format(execution_id))
return False
def resume_workflow(self, execution_id):
"""Resumes a workflow that is currently paused.
Args:
execution_id (str): The execution id of the workflow.
"""
workflow_status = executiondb.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status and workflow_status.status == WorkflowStatusEnum.paused:
saved_state = executiondb.execution_db.session.query(
SavedWorkflow).filter_by(
workflow_execution_id=execution_id).first()
workflow = executiondb.execution_db.session.query(
Workflow).filter_by(id=workflow_status.workflow_id).first()
workflow._execution_id = execution_id
WalkoffEvent.WorkflowResumed.send(workflow)
start = saved_state.action_id if saved_state else workflow.start
self.execute_workflow(workflow.id,
execution_id_in=execution_id,
start=start,
resume=True)
return True
else:
logger.warning(
'Cannot resume workflow {0}. Invalid key, or workflow not paused.'
.format(execution_id))
return False
def abort_workflow(self, execution_id):
"""Abort a workflow.
Args:
execution_id (str): The execution id of the workflow.
"""
workflow_status = executiondb.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status:
if workflow_status.status in [
WorkflowStatusEnum.pending, WorkflowStatusEnum.paused,
WorkflowStatusEnum.awaiting_data
]:
workflow = walkoff.coredb.devicedb.device_db.session.query(
Workflow).filter_by(
id=workflow_status.workflow_id).first()
if workflow is not None:
WalkoffEvent.WorkflowAborted.send({
'execution_id': execution_id,
'id': workflow_status.workflow_id,
'name': workflow.name
})
elif workflow_status.status == WorkflowStatusEnum.running:
self.manager.abort_workflow(execution_id)
return True
else:
logger.warning(
'Cannot resume workflow {0}. Invalid key, or workflow already shutdown.'
.format(execution_id))
return False
def resume_trigger_step(self, execution_id, data_in, arguments=None):
"""Resumes a workflow awaiting trigger data, if the conditions are met.
Args:
execution_id (str): The execution ID of the workflow
data_in (dict): The data to send to the trigger
arguments (list[Argument], optional): Optional list of new Arguments for the trigger action.
Defaults to None.
"""
saved_state = executiondb.execution_db.session.query(
SavedWorkflow).filter_by(
workflow_execution_id=execution_id).first()
workflow = executiondb.execution_db.session.query(Workflow).filter_by(
id=saved_state.workflow_id).first()
workflow._execution_id = execution_id
executed = False
exec_action = None
for action in workflow.actions:
if action.id == saved_state.action_id:
exec_action = action
executed = action.execute_trigger(data_in,
saved_state.accumulator)
break
if executed:
WalkoffEvent.TriggerActionTaken.send(
exec_action, data={'workflow_execution_id': execution_id})
self.execute_workflow(workflow.id,
execution_id_in=execution_id,
start=saved_state.action_id,
start_arguments=arguments,
resume=True)
return True
else:
WalkoffEvent.TriggerActionNotTaken.send(
exec_action, data={'workflow_execution_id': execution_id})
return False
@staticmethod
def get_waiting_workflows():
"""Gets a list of the execution IDs of workflows currently awaiting data to be sent to a trigger.
Returns:
A list of execution IDs of workflows currently awaiting data to be sent to a trigger.
"""
executiondb.execution_db.session.expire_all()
wf_statuses = executiondb.execution_db.session.query(
WorkflowStatus).filter_by(
status=WorkflowStatusEnum.awaiting_data).all()
return [str(wf_status.execution_id) for wf_status in wf_statuses]
def get_workflow_status(self, execution_id):
"""Gets the current status of a workflow by its execution ID
Args:
execution_id (str): The execution ID of the workflow
Returns:
The status of the workflow
"""
workflow_status = executiondb.execution_db.session.query(
WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status:
return workflow_status.status
else:
logger.error("Key {} does not exist in database.").format(
execution_id)
return 0
class MultiprocessedExecutor(object):
def __init__(self, cache, config):
"""Initializes a multiprocessed executor, which will handle the execution of workflows.
"""
self.threading_is_initialized = False
self.id = "controller"
self.pids = None
self.workflows_executed = 0
self.ctx = None # TODO: Test if you can always use the singleton
self.auth = None
self.zmq_workflow_comm = None
self.receiver = None
self.receiver_thread = None
self.cache = cache
self.config = config
self.execution_db = ExecutionDatabase.instance
self.results_sender = None
key = PrivateKey(walkoff.config.Config.SERVER_PRIVATE_KEY[:nacl.bindings.crypto_box_SECRETKEYBYTES])
worker_key = PrivateKey(
walkoff.config.Config.CLIENT_PRIVATE_KEY[:nacl.bindings.crypto_box_SECRETKEYBYTES]).public_key
self.__box = Box(key, worker_key)
def initialize_threading(self, app, pids=None):
"""Initialize the multiprocessing communication threads, allowing for parallel execution of workflows.
Args:
app (FlaskApp): The current_app object
pids (list[Process], optional): Optional list of spawned processes. Defaults to None
"""
if walkoff.config.Config.SEPARATE_RECEIVER:
pass
self.pids = pids
if 'zmq' in [walkoff.config.Config.WORKFLOW_COMMUNICATION_HANDLER,
walkoff.config.Config.WORKFLOW_RESULTS_HANDLER]:
# Only run the threadauthenticator if ZMQ is in use
self.ctx = zmq.Context.instance()
self.auth = ThreadAuthenticator()
self.auth.start()
# TODO: self.auth.allow('127.0.0.1')
self.auth.configure_curve(domain='*', location=walkoff.config.Config.ZMQ_PUBLIC_KEYS_PATH)
self.zmq_workflow_comm = make_communication_sender()
with app.app_context():
data = {'execution_db': current_app.running_context.execution_db}
self.results_sender = make_results_sender(**data)
if not walkoff.config.Config.SEPARATE_RECEIVER:
data = {'current_app': app}
self.receiver = make_results_receiver(**data)
self.receiver_thread = threading.Thread(target=self.receiver.receive_results)
self.receiver_thread.start()
self.threading_is_initialized = True
logger.debug('Controller threading initialized')
def wait_and_reset(self, num_workflows):
"""Waits for all of the workflows to be completed
Args:
num_workflows (int): The number of workflows to wait for
"""
timeout = 0
shutdown = 10
while timeout < shutdown:
if self.receiver is not None and num_workflows == self.receiver.workflows_executed:
break
timeout += 0.1
gevent.sleep(0.1)
assert (num_workflows == self.receiver.workflows_executed)
self.receiver.workflows_executed = 0
def shutdown_pool(self):
"""Shuts down the threadpool"""
if self.zmq_workflow_comm:
self.zmq_workflow_comm.send_exit_to_workers()
if not walkoff.config.Config.SEPARATE_WORKERS:
shutdown_procs(self.pids)
if self.receiver_thread:
self.receiver.thread_exit = True
self.receiver_thread.join(timeout=1)
self.threading_is_initialized = False
logger.debug('Controller thread pool shutdown')
if self.auth:
self.auth.stop()
if self.ctx:
self.ctx.destroy()
self.cleanup_threading()
self.execution_db.tear_down()
return
def cleanup_threading(self):
"""Once the threadpool has been shutdown, clear out all of the data structures used in the pool"""
self.pids = []
self.receiver_thread = None
self.workflows_executed = 0
self.threading_is_initialized = False
self.zmq_workflow_comm = None
self.receiver = None
def execute_workflow(self, workflow_id, execution_id_in=None, start=None, start_arguments=None, resume=False,
environment_variables=None, user=None):
"""Executes a workflow
Args:
workflow_id (Workflow): The Workflow to be executed.
execution_id_in (UUID, optional): The optional execution ID to provide for the workflow. Should only be
used (and is required) when resuming a workflow. Must be valid UUID4. Defaults to None.
start (UUID, optional): The ID of the first, or starting action. Defaults to None.
start_arguments (list[Argument]): The arguments to the starting action of the workflow. Defaults to None.
resume (bool, optional): Optional boolean to resume a previously paused workflow. Defaults to False.
environment_variables (list[EnvironmentVariable]): Optional list of environment variables to pass into
the workflow. These will not be persistent.
user (str, Optional): The username of the user who requested that this workflow be executed. Defaults
to None.
Returns:
(UUID): The execution ID of the Workflow.
"""
workflow = self.execution_db.session.query(Workflow).filter_by(id=workflow_id).first()
if not workflow:
logger.error('Attempted to execute workflow {} which does not exist'.format(execution_id_in))
return None, 'Attempted to execute workflow which does not exist'
execution_id = execution_id_in if execution_id_in else str(uuid.uuid4())
if start is not None:
logger.info('User {0} executing workflow {1} (id={2}) with starting action {3}'.format(
user, workflow.name, workflow.id, start))
else:
logger.info('User {0} executing workflow {1} (id={2}) with default starting action {3}'.format(
user, workflow.name, workflow.id, workflow.start))
workflow_data = {'execution_id': execution_id, 'id': str(workflow.id), 'name': workflow.name}
data = {}
if user:
data['user'] = user
self._log_and_send_event(WalkoffEvent.WorkflowExecutionPending, sender=workflow_data, workflow=workflow,
data=data)
self.__add_workflow_to_queue(workflow.id, execution_id, start, start_arguments, resume, environment_variables,
user)
self._log_and_send_event(WalkoffEvent.SchedulerJobExecuted, data=data)
return execution_id
def __add_workflow_to_queue(self, workflow_id, workflow_execution_id, start=None, start_arguments=None,
resume=False, environment_variables=None, user=None):
message = self.results_sender.create_workflow_request_message(workflow_id, workflow_execution_id, start,
start_arguments, resume, environment_variables,
user)
self.cache.lpush("request_queue", self.__box.encrypt(message))
def pause_workflow(self, execution_id, user=None):
"""Pauses a workflow that is currently executing.
Args:
execution_id (UUID): The execution id of the workflow.
Returns:
(bool): True if Workflow successfully paused, False otherwise
"""
logger.info('User {0} pausing workflow {1}'.format(user, execution_id))
workflow_status = self.execution_db.session.query(WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status and workflow_status.status == WorkflowStatusEnum.running:
self.zmq_workflow_comm.pause_workflow(execution_id)
return True
else:
logger.warning('Cannot pause workflow {0}. Invalid key, or workflow not running.'.format(execution_id))
return False
def resume_workflow(self, execution_id, user=None):
"""Resumes a workflow that is currently paused.
Args:
execution_id (UUID): The execution id of the workflow.
user (str, Optional): The username of the user who requested that this workflow be resumed. Defaults
to None.
Returns:
(bool): True if workflow successfully resumed, False otherwise
"""
logger.info('User {0} resuming workflow {1}'.format(user, execution_id))
workflow_status = self.execution_db.session.query(WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status and workflow_status.status == WorkflowStatusEnum.paused:
saved_state = self.execution_db.session.query(SavedWorkflow).filter_by(
workflow_execution_id=execution_id).first()
workflow = self.execution_db.session.query(Workflow).filter_by(id=workflow_status.workflow_id).first()
data = {"execution_id": execution_id}
if user:
data['user'] = user
self._log_and_send_event(WalkoffEvent.WorkflowResumed, sender=workflow, data=data)
start = saved_state.action_id if saved_state else workflow.start
self.execute_workflow(workflow.id, execution_id_in=execution_id, start=start, resume=True, user=user)
return True
else:
logger.warning('Cannot resume workflow {0}. Invalid key, or workflow not paused.'.format(execution_id))
return False
def abort_workflow(self, execution_id, user=None):
"""Abort a workflow
Args:
execution_id (UUID): The execution id of the workflow.
user (str, Optional): The username of the user who requested that this workflow be aborted. Defaults
to None.
Returns:
(bool): True if successfully aborted workflow, False otherwise
"""
logger.info('User {0} aborting workflow {1}'.format(user, execution_id))
workflow_status = self.execution_db.session.query(WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status:
if workflow_status.status in [WorkflowStatusEnum.pending, WorkflowStatusEnum.paused,
WorkflowStatusEnum.awaiting_data]:
workflow = self.execution_db.session.query(Workflow).filter_by(id=workflow_status.workflow_id).first()
if workflow is not None:
data = {}
if user:
data['user'] = user
self._log_and_send_event(WalkoffEvent.WorkflowAborted,
sender={'execution_id': execution_id, 'id': workflow_status.workflow_id,
'name': workflow.name}, workflow=workflow, data=data)
elif workflow_status.status == WorkflowStatusEnum.running:
self.zmq_workflow_comm.abort_workflow(execution_id)
return True
else:
logger.warning(
'Cannot resume workflow {0}. Invalid key, or workflow already shutdown.'.format(execution_id))
return False
def resume_trigger_step(self, execution_id, data_in, arguments=None, user=None):
"""Resumes a workflow awaiting trigger data, if the conditions are met.
Args:
execution_id (UUID): The execution ID of the workflow
data_in (dict): The data to send to the trigger
arguments (list[Argument], optional): Optional list of new Arguments for the trigger action.
Defaults to None.
user (str, Optional): The username of the user who requested that this workflow be resumed. Defaults
to None.
Returns:
(bool): True if successfully resumed trigger step, false otherwise
"""
logger.info('User {0} resuming workflow {1} from trigger'.format(user, execution_id))
saved_state = self.execution_db.session.query(SavedWorkflow).filter_by(
workflow_execution_id=execution_id).first()
workflow = self.execution_db.session.query(Workflow).filter_by(id=saved_state.workflow_id).first()
action_execution_strategy = make_execution_strategy(
self.config,
RestrictedWorkflowContext.from_workflow(workflow, execution_id)
)
accumulator = make_accumulator(execution_id)
executed = False
exec_action = None
for action in workflow.actions:
if action.id == saved_state.action_id:
exec_action = action
executed = action.execute_trigger(action_execution_strategy, data_in, accumulator)
break
data = {'workflow_execution_id': execution_id}
if user:
data['user'] = user
if executed:
self._log_and_send_event(WalkoffEvent.TriggerActionTaken, sender=exec_action,
data={'workflow_execution_id': execution_id}, workflow=workflow)
self.execute_workflow(workflow.id, execution_id_in=execution_id, start=str(saved_state.action_id),
start_arguments=arguments, resume=True, user=user)
return True
else:
self._log_and_send_event(WalkoffEvent.TriggerActionNotTaken, sender=exec_action,
data={'workflow_execution_id': execution_id}, workflow=workflow)
return False
def get_waiting_workflows(self):
"""Gets a list of the execution IDs of workflows currently awaiting data to be sent to a trigger.
Returns:
(list[UUID]): A list of execution IDs of workflows currently awaiting data to be sent to a trigger.
"""
self.execution_db.session.expire_all()
wf_statuses = self.execution_db.session.query(WorkflowStatus).filter_by(
status=WorkflowStatusEnum.awaiting_data).all()
return [str(wf_status.execution_id) for wf_status in wf_statuses]
def get_workflow_status(self, execution_id):
"""Gets the current status of a workflow by its execution ID
Args:
execution_id (UUID): The execution ID of the workflow
Returns:
(int): The status of the workflow
"""
workflow_status = self.execution_db.session.query(WorkflowStatus).filter_by(execution_id=execution_id).first()
if workflow_status:
return workflow_status.status
else:
logger.error("Workflow execution id {} does not exist in WorkflowStatus table.").format(execution_id)
return 0
def _log_and_send_event(self, event, sender=None, data=None, workflow=None):
sender = sender or self
self.results_sender.handle_event(workflow, sender, event=event, data=data)