class W5Timer(object):
_instance_lock = threading.Lock()
def __init__(self):
self.scheduler = None
def __new__(cls, *args, **kwargs):
if not hasattr(W5Timer, "_instance"):
with W5Timer._instance_lock:
if not hasattr(W5Timer, "_instance"):
W5Timer._instance = object.__new__(cls)
return W5Timer._instance
def create_scheduler(self):
self.scheduler = GeventScheduler()
def start(self):
self.scheduler.start()
def shutdown(self):
self.scheduler.shutdown()
def pause(self, uuid):
self.scheduler.pause_job(uuid)
def pause_all(self):
self.scheduler.pause()
def resume(self, uuid):
self.scheduler.resume_job(uuid)
def resume_all(self):
self.scheduler.resume()
def remove_job(self, uuid):
self.scheduler.remove_job(uuid)
def get_jobs(self):
return self.scheduler.get_jobs()
def add_date(self, run_date=None, uuid=None, timer_uuid=None):
self.scheduler.add_job(
auto_execute,
'date',
run_date=run_date,
id=timer_uuid,
args=(uuid,)
)
def update_date(self, uuid, run_date=None):
self.scheduler.reschedule_job(
uuid,
trigger='date',
run_date=run_date
)
def add_interval(self, t, interval, uuid=None, timer_uuid=None, start_date=None, end_date=None, jitter=0):
if t == "seconds":
self.scheduler.add_job(
auto_execute,
'interval',
seconds=interval,
start_date=start_date,
end_date=end_date,
jitter=jitter,
id=timer_uuid,
args=(uuid,)
)
elif t == "minutes":
self.scheduler.add_job(
auto_execute,
'interval',
minutes=interval,
start_date=start_date,
end_date=end_date,
jitter=jitter,
id=timer_uuid,
args=(uuid,)
)
elif t == "hours":
self.scheduler.add_job(
auto_execute,
'interval',
hours=interval,
start_date=start_date,
end_date=end_date,
jitter=jitter,
id=timer_uuid,
args=(uuid,)
)
elif t == "days":
self.scheduler.add_job(
auto_execute,
'interval',
days=interval,
start_date=start_date,
end_date=end_date,
jitter=jitter,
id=timer_uuid,
args=(uuid,)
)
elif t == "weeks":
self.scheduler.add_job(
auto_execute,
'interval',
weeks=interval,
start_date=start_date,
end_date=end_date,
jitter=jitter,
id=timer_uuid,
args=(uuid,)
)
def update_interval(self, uuid, t, interval, start_date=None, end_date=None, jitter=0):
if t == "seconds":
self.scheduler.reschedule_job(
uuid,
trigger="interval",
seconds=interval,
start_date=start_date,
end_date=end_date,
jitter=jitter
)
elif t == "minutes":
self.scheduler.reschedule_job(
uuid,
trigger="interval",
minutes=interval,
start_date=start_date,
end_date=end_date,
jitter=jitter
)
elif t == "hours":
self.scheduler.reschedule_job(
uuid,
trigger="interval",
hours=interval,
start_date=start_date,
end_date=end_date,
jitter=jitter
)
elif t == "days":
self.scheduler.reschedule_job(
uuid,
trigger="interval",
days=interval,
start_date=start_date,
end_date=end_date,
jitter=jitter
)
elif t == "weeks":
self.scheduler.reschedule_job(
uuid,
trigger="interval",
weeks=interval,
start_date=start_date,
end_date=end_date,
jitter=jitter
)
def add_cron(self, cron, uuid=None, timer_uuid=None, start_date=None, end_date=None, jitter=0):
self.scheduler.add_job(
auto_execute,
CronTrigger.from_crontab(cron),
start_date=start_date,
end_date=end_date,
jitter=jitter,
id=timer_uuid,
args=(uuid,)
)
def update_cron(self, uuid, cron, start_date=None, end_date=None, jitter=0):
values = cron.split()
if len(values) != 5:
raise ValueError('Wrong number of fields; got {}, expected 5'.format(len(values)))
self.scheduler.reschedule_job(
uuid,
None,
"cron",
minute=values[0],
hour=values[1],
day=values[2],
month=values[3],
day_of_week=values[4],
start_date=start_date,
end_date=end_date,
jitter=jitter
)
class Scheduler(object):
def __init__(self):
self.scheduler = GeventScheduler()
self.scheduler.add_listener(self.__scheduler_listener(),
EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN
| EVENT_SCHEDULER_PAUSED | EVENT_SCHEDULER_RESUMED
| EVENT_JOB_ADDED | EVENT_JOB_REMOVED
| EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
self.id = 'controller'
self.app = None
def schedule_workflows(self, task_id, executable, workflow_ids, trigger):
"""
Schedules a workflow for execution
Args:
task_id (int): Id of the scheduled task
executable (func): A callable to execute must take in one argument -- a workflow id
workflow_ids (iterable(str)): An iterable of workflow ids
trigger (Trigger): The trigger to use for this scheduled task
"""
def execute(id_):
with self.app.app_context():
executable(id_)
for workflow_id in workflow_ids:
self.scheduler.add_job(execute, args=(workflow_id,),
id=construct_task_id(task_id, workflow_id),
trigger=trigger, replace_existing=True)
def get_all_scheduled_workflows(self):
"""
Gets all the scheduled workflows
Returns:
(dict{str: list[str]}) A dict of task_id to workflow execution ids
"""
tasks = {}
for job in self.scheduler.get_jobs():
task, workflow_execution_id = split_task_id(job.id)
if task not in tasks:
tasks[task] = [workflow_execution_id]
else:
tasks[task].append(workflow_execution_id)
return tasks
def get_scheduled_workflows(self, task_id):
"""
Gets all the scheduled worfklows for a given task id
Args:
task_id (str): The task id
Returns:
(list[str]) A list fo workflow execution id associated with this task id
"""
tasks = []
for job in self.scheduler.get_jobs():
task, workflow_execution_id = split_task_id(job.id)
if task == task_id:
tasks.append(workflow_execution_id)
return tasks
def update_workflows(self, task_id, trigger):
"""
Updates the workflows for a given task id to use a different trigger
Args:
task_id (str|int): The task id to update
trigger (Trigger): The new trigger to use
"""
existing_tasks = {construct_task_id(task_id, workflow_execution_id) for workflow_execution_id in
self.get_scheduled_workflows(task_id)}
for job_id in existing_tasks:
self.scheduler.reschedule_job(job_id=job_id, trigger=trigger)
def unschedule_workflows(self, task_id, workflow_execution_ids):
"""
Unschedules a workflow
Args:
task_id (str|int): The task ID to unschedule
workflow_execution_ids (list[str]): The list of workflow execution IDs to update
"""
for workflow_execution_id in workflow_execution_ids:
try:
self.scheduler.remove_job(construct_task_id(task_id, workflow_execution_id))
except JobLookupError:
logger.warning('Cannot delete task {}. '
'No task found in scheduler'.format(construct_task_id(task_id, workflow_execution_id)))
def start(self):
"""Starts the scheduler for active execution. This function must be called before any workflows are executed.
Returns:
The state of the scheduler if successful, error message if scheduler is in "stopped" state.
"""
if self.scheduler.state == STATE_STOPPED:
logger.info('Starting scheduler')
self.scheduler.start()
else:
logger.warning('Cannot start scheduler. Scheduler is already running or is paused')
return "Scheduler already running."
return self.scheduler.state
def stop(self, wait=True):
"""Stops active execution.
Args:
wait (bool, optional): Boolean to synchronously or asynchronously wait for the scheduler to shutdown.
Default is True.
Returns:
The state of the scheduler if successful, error message if scheduler is already in "stopped" state.
"""
if self.scheduler.state != STATE_STOPPED:
logger.info('Stopping scheduler')
self.scheduler.shutdown(wait=wait)
else:
logger.warning('Cannot stop scheduler. Scheduler is already stopped')
return "Scheduler already stopped."
return self.scheduler.state
def pause(self):
"""Pauses active execution.
Returns:
The state of the scheduler if successful, error message if scheduler is not in the "running" state.
"""
if self.scheduler.state == STATE_RUNNING:
logger.info('Pausing scheduler')
self.scheduler.pause()
elif self.scheduler.state == STATE_PAUSED:
logger.warning('Cannot pause scheduler. Scheduler is already paused')
return "Scheduler already paused."
elif self.scheduler.state == STATE_STOPPED:
logger.warning('Cannot pause scheduler. Scheduler is stopped')
return "Scheduler is in STOPPED state and cannot be paused."
return self.scheduler.state
def resume(self):
"""Resumes active execution.
Returns:
The state of the scheduler if successful, error message if scheduler is not in the "paused" state.
"""
if self.scheduler.state == STATE_PAUSED:
logger.info('Resuming scheduler')
self.scheduler.resume()
else:
logger.warning("Scheduler is not in PAUSED state and cannot be resumed.")
return "Scheduler is not in PAUSED state and cannot be resumed."
return self.scheduler.state
def pause_workflows(self, task_id, workflow_execution_ids):
"""
Pauses some workflows associated with a task
Args:
task_id (int|str): The id of the task to pause
workflow_execution_ids (list[str]): The list of workflow execution IDs to pause
"""
for workflow_execution_id in workflow_execution_ids:
job_id = construct_task_id(task_id, workflow_execution_id)
try:
self.scheduler.pause_job(job_id=job_id)
logger.info('Paused job {0}'.format(job_id))
except JobLookupError:
logger.warning('Cannot pause scheduled workflow {}. Workflow ID not found'.format(job_id))
def resume_workflows(self, task_id, workflow_execution_ids):
"""
Resumes some workflows associated with a task
Args:
task_id (int|str): The id of the task to pause
workflow_execution_ids (list[str]): The list of workflow execution IDs to resume
"""
for workflow_execution_id in workflow_execution_ids:
job_id = construct_task_id(task_id, workflow_execution_id)
try:
self.scheduler.resume_job(job_id=job_id)
logger.info('Resumed job {0}'.format(job_id))
except JobLookupError:
logger.warning('Cannot resume scheduled workflow {}. Workflow ID not found'.format(job_id))
def __scheduler_listener(self):
event_selector_map = {EVENT_SCHEDULER_START: WalkoffEvent.SchedulerStart,
EVENT_SCHEDULER_SHUTDOWN: WalkoffEvent.SchedulerShutdown,
EVENT_SCHEDULER_PAUSED: WalkoffEvent.SchedulerPaused,
EVENT_SCHEDULER_RESUMED: WalkoffEvent.SchedulerResumed,
EVENT_JOB_ADDED: WalkoffEvent.SchedulerJobAdded,
EVENT_JOB_REMOVED: WalkoffEvent.SchedulerJobRemoved,
EVENT_JOB_EXECUTED: WalkoffEvent.SchedulerJobExecuted,
EVENT_JOB_ERROR: WalkoffEvent.SchedulerJobError}
def event_selector(event):
try:
event = event_selector_map[event.code]
event.send(self)
except KeyError: # pragma: no cover
logger.error('Unknown event sent triggered in scheduler {}'.format(event))
return event_selector
class Scheduler(object):
def __init__(self):
self.scheduler = GeventScheduler()
self.scheduler.add_listener(self.__scheduler_listener(),
EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN
| EVENT_SCHEDULER_PAUSED | EVENT_SCHEDULER_RESUMED
| EVENT_JOB_ADDED | EVENT_JOB_REMOVED
| EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
self.id = 'controller'
self.app = None
def schedule_workflows(self, task_id, executable, workflow_ids, trigger):
"""
Schedules a workflow for execution
Args:
task_id (int): Id of the scheduled task
executable (func): A callable to execute must take in one argument -- a workflow id
workflow_ids (iterable(str)): An iterable of workflow ids
trigger (Trigger): The trigger to use for this scheduled task
"""
def execute(id_):
with self.app.app_context():
executable(id_)
for workflow_id in workflow_ids:
self.scheduler.add_job(execute, args=(workflow_id,),
id=construct_task_id(task_id, workflow_id),
trigger=trigger, replace_existing=True)
def get_all_scheduled_workflows(self):
"""
Gets all the scheduled workflows
Returns:
(dict{str: list[str]}) A dict of task_id to workflow execution ids
"""
tasks = {}
for job in self.scheduler.get_jobs():
task, workflow_execution_id = split_task_id(job.id)
if task not in tasks:
tasks[task] = [workflow_execution_id]
else:
tasks[task].append(workflow_execution_id)
return tasks
def get_scheduled_workflows(self, task_id):
"""
Gets all the scheduled worfklows for a given task id
Args:
task_id (str): The task id
Returns:
(list[str]) A list fo workflow execution id associated with this task id
"""
tasks = []
for job in self.scheduler.get_jobs():
task, workflow_execution_id = split_task_id(job.id)
if task == task_id:
tasks.append(workflow_execution_id)
return tasks
def update_workflows(self, task_id, trigger):
"""
Updates the workflows for a given task id to use a different trigger
Args:
task_id (str|int): The task id to update
trigger (Trigger): The new trigger to use
"""
existing_tasks = {construct_task_id(task_id, workflow_execution_id) for workflow_execution_id in
self.get_scheduled_workflows(task_id)}
for job_id in existing_tasks:
self.scheduler.reschedule_job(job_id=job_id, trigger=trigger)
def unschedule_workflows(self, task_id, workflow_execution_ids):
"""
Unschedules a workflow
Args:
task_id (str|int): The task ID to unschedule
workflow_execution_ids (list[str]): The list of workflow execution IDs to update
"""
for workflow_execution_id in workflow_execution_ids:
try:
self.scheduler.remove_job(construct_task_id(task_id, workflow_execution_id))
except JobLookupError:
logger.warning('Cannot delete task {}. '
'No task found in scheduler'.format(construct_task_id(task_id, workflow_execution_id)))
def start(self):
"""Starts the scheduler for active execution. This function must be called before any workflows are executed.
Returns:
The state of the scheduler if successful, error message if scheduler is in "stopped" state.
"""
if self.scheduler.state == STATE_STOPPED:
logger.info('Starting scheduler')
self.scheduler.start()
else:
logger.warning('Cannot start scheduler. Scheduler is already running or is paused')
return "Scheduler already running."
return self.scheduler.state
def stop(self, wait=True):
"""Stops active execution.
Args:
wait (bool, optional): Boolean to synchronously or asynchronously wait for the scheduler to shutdown.
Default is True.
Returns:
The state of the scheduler if successful, error message if scheduler is already in "stopped" state.
"""
if self.scheduler.state != STATE_STOPPED:
logger.info('Stopping scheduler')
self.scheduler.shutdown(wait=wait)
else:
logger.warning('Cannot stop scheduler. Scheduler is already stopped')
return "Scheduler already stopped."
return self.scheduler.state
def pause(self):
"""Pauses active execution.
Returns:
The state of the scheduler if successful, error message if scheduler is not in the "running" state.
"""
if self.scheduler.state == STATE_RUNNING:
logger.info('Pausing scheduler')
self.scheduler.pause()
elif self.scheduler.state == STATE_PAUSED:
logger.warning('Cannot pause scheduler. Scheduler is already paused')
return "Scheduler already paused."
elif self.scheduler.state == STATE_STOPPED:
logger.warning('Cannot pause scheduler. Scheduler is stopped')
return "Scheduler is in STOPPED state and cannot be paused."
return self.scheduler.state
def resume(self):
"""Resumes active execution.
Returns:
The state of the scheduler if successful, error message if scheduler is not in the "paused" state.
"""
if self.scheduler.state == STATE_PAUSED:
logger.info('Resuming scheduler')
self.scheduler.resume()
else:
logger.warning("Scheduler is not in PAUSED state and cannot be resumed.")
return "Scheduler is not in PAUSED state and cannot be resumed."
return self.scheduler.state
def pause_workflows(self, task_id, workflow_execution_ids):
"""
Pauses some workflows associated with a task
Args:
task_id (int|str): The id of the task to pause
workflow_execution_ids (list[str]): The list of workflow execution IDs to pause
"""
for workflow_execution_id in workflow_execution_ids:
job_id = construct_task_id(task_id, workflow_execution_id)
try:
self.scheduler.pause_job(job_id=job_id)
logger.info('Paused job {0}'.format(job_id))
except JobLookupError:
logger.warning('Cannot pause scheduled workflow {}. Workflow ID not found'.format(job_id))
def resume_workflows(self, task_id, workflow_execution_ids):
"""
Resumes some workflows associated with a task
Args:
task_id (int|str): The id of the task to pause
workflow_execution_ids (list[str]): The list of workflow execution IDs to resume
"""
for workflow_execution_id in workflow_execution_ids:
job_id = construct_task_id(task_id, workflow_execution_id)
try:
self.scheduler.resume_job(job_id=job_id)
logger.info('Resumed job {0}'.format(job_id))
except JobLookupError:
logger.warning('Cannot resume scheduled workflow {}. Workflow ID not found'.format(job_id))
def __scheduler_listener(self):
event_selector_map = {EVENT_SCHEDULER_START: WalkoffEvent.SchedulerStart,
EVENT_SCHEDULER_SHUTDOWN: WalkoffEvent.SchedulerShutdown,
EVENT_SCHEDULER_PAUSED: WalkoffEvent.SchedulerPaused,
EVENT_SCHEDULER_RESUMED: WalkoffEvent.SchedulerResumed,
EVENT_JOB_ADDED: WalkoffEvent.SchedulerJobAdded,
EVENT_JOB_REMOVED: WalkoffEvent.SchedulerJobRemoved,
EVENT_JOB_EXECUTED: WalkoffEvent.SchedulerJobExecuted,
EVENT_JOB_ERROR: WalkoffEvent.SchedulerJobError}
def event_selector(event):
try:
event = event_selector_map[event.code]
event.send(self)
except KeyError: # pragma: no cover
logger.error('Unknown event sent triggered in scheduler {}'.format(event))
return event_selector
