def _queue_upcoming_tasks(self, schedule_snapshot):
upcoming_queue = TaskQueue()
upcoming_task_times = self._get_upcoming_task_times(schedule_snapshot)
for entry, task_times in upcoming_task_times.items():
task_id = None
pri = entry.priority
action = entry.action
for t in task_times:
upcoming_queue.enter(t, pri, action, entry.name, task_id)
return upcoming_queue
def _queue_pending_tasks(self, schedule_snapshot):
pending_queue = TaskQueue()
for entry in schedule_snapshot:
task_time = self._take_pending_task_time(entry)
self._cancel_if_completed(entry)
if task_time is None:
continue
task_id = entry.get_next_task_id()
entry.save(update_fields=("next_task_id",))
pri = entry.priority
action = entry.action
pending_queue.enter(task_time, pri, action, entry.name, task_id)
return pending_queue
def __init__(self):
threading.Thread.__init__(self)
self.timefn = utils.timefn
self.delayfn = utils.delayfn
self.task_queue = TaskQueue()
# scheduler looks ahead `interval_multiplier` times the shortest
# interval in the schedule in order to keep memory-usage low
self.interval_multiplier = 10
self.name = "Scheduler"
self.running = False
self.interrupt_flag = threading.Event()
# Cache the currently running task state
self.entry = None # ScheduleEntry that created the current task
self.task = None # Task object describing current task
self.task_result = None # TaskResult object for current task
def load_queues(self, json_profile):
for k, json_queue in json_profile['queues'].items():
q = TaskQueue({'name': k, 'stop_order': json_queue['stop_order']})
self.add_queue(q)
self.arr_queues = list(
sorted(self.arr_queues, key=lambda x: x.stop_order))
class Scheduler(threading.Thread):
"""A memory-friendly task scheduler."""
def __init__(self):
threading.Thread.__init__(self)
self.timefn = utils.timefn
self.delayfn = utils.delayfn
self.task_queue = TaskQueue()
# scheduler looks ahead `interval_multiplier` times the shortest
# interval in the schedule in order to keep memory-usage low
self.interval_multiplier = 10
self.name = "Scheduler"
self.running = False
self.interrupt_flag = threading.Event()
# Cache the currently running task state
self.entry = None # ScheduleEntry that created the current task
self.task = None # Task object describing current task
self.task_result = None # TaskResult object for current task
@property
def schedule(self):
"""An updated view of the current schedule"""
return ScheduleEntry.objects.filter(is_active=True).all()
@property
def schedule_has_entries(self):
"""True if active events exist in the schedule, otherwise False."""
return ScheduleEntry.objects.filter(is_active=True).exists()
@staticmethod
def cancel(entry):
"""Remove an entry from the scheduler without deleting it."""
entry.is_active = False
entry.save(update_fields=("is_active",))
def stop(self):
"""Complete the current task, then return control."""
self.interrupt_flag.set()
def start(self):
"""Run the scheduler in its own thread and return control."""
threading.Thread.start(self)
def run(self, blocking=True):
"""Run the scheduler in the current thread.
:param blocking: block until stopped or return control after each task
"""
if blocking:
try:
Path(settings.SCHEDULER_HEALTHCHECK_FILE).unlink()
except FileNotFoundError:
pass
try:
while True:
with minimum_duration(blocking):
self._consume_schedule(blocking)
if not blocking or self.interrupt_flag.is_set():
logger.info("scheduler interrupted")
break
except Exception as err:
logger.warning("scheduler dead")
logger.exception(err)
if settings.IN_DOCKER:
Path(settings.SCHEDULER_HEALTHCHECK_FILE).touch()
def _consume_schedule(self, blocking):
while self.schedule_has_entries:
with minimum_duration(blocking):
self.running = True
schedule_snapshot = self.schedule
pending_task_queue = self._queue_tasks(schedule_snapshot)
self._consume_task_queue(pending_task_queue)
if not blocking or self.interrupt_flag.is_set():
break
else:
self.task_queue.clear()
if self.running:
logger.info("all scheduled tasks completed")
self.running = False
def _queue_tasks(self, schedule_snapshot):
pending_task_queue = self._queue_pending_tasks(schedule_snapshot)
self.task_queue = self._queue_upcoming_tasks(schedule_snapshot)
return pending_task_queue
def _consume_task_queue(self, pending_task_queue):
for task in pending_task_queue.to_list():
entry_name = task.schedule_entry_name
self.task = task
self.entry = ScheduleEntry.objects.get(name=entry_name)
self._initialize_task_result()
started = timezone.now()
status, detail = self._call_task_action()
finished = timezone.now()
self._finalize_task_result(started, finished, status, detail)
def _initialize_task_result(self):
"""Initalize an 'in-progress' result so it exists when action runs."""
tid = self.task.task_id
self.task_result = TaskResult(schedule_entry=self.entry, task_id=tid)
self.task_result.save()
def _call_task_action(self):
entry_name = self.task.schedule_entry_name
task_id = self.task.task_id
try:
logger.debug("running task {}/{}".format(entry_name, task_id))
detail = self.task.action_fn(entry_name, task_id)
self.delayfn(0) # let other threads run
status = "success"
if not isinstance(detail, str):
detail = ""
except Exception as err:
detail = str(err)
logger.exception("action failed: {}".format(detail))
status = "failure"
return status, detail[:MAX_DETAIL_LEN]
def _finalize_task_result(self, started, finished, status, detail):
tr = self.task_result
tr.started = started
tr.finished = finished
tr.duration = finished - started
tr.status = status
tr.detail = detail
tr.save()
if self.entry.callback_url:
try:
logger.debug("Trying callback")
context = {"request": self.entry.request}
result_json = TaskResultSerializer(tr, context=context).data
verify_ssl = settings.CALLBACK_SSL_VERIFICATION
if settings.CALLBACK_SSL_VERIFICATION:
if settings.PATH_TO_VERIFY_CERT != "":
verify_ssl = settings.PATH_TO_VERIFY_CERT
logger.debug(settings.CALLBACK_AUTHENTICATION)
if settings.CALLBACK_AUTHENTICATION == "OAUTH":
client = oauth.get_oauth_client()
headers = {"Content-Type": "application/json"}
response = client.post(
self.entry.callback_url,
data=json.dumps(result_json),
headers=headers,
verify=verify_ssl,
)
self._callback_response_handler(client, response)
else:
token = self.entry.owner.auth_token
headers = {"Authorization": "Token " + str(token)}
requests_futures_session.post(
self.entry.callback_url,
json=result_json,
background_callback=self._callback_response_handler,
headers=headers,
verify=verify_ssl,
)
except Exception as err:
logger.error(str(err))
@staticmethod
def _callback_response_handler(sess, resp):
if resp.ok:
logger.info("POSTed to {}".format(resp.url))
else:
msg = "Failed to POST to {}: {}"
logger.warning(msg.format(resp.url, resp.reason))
def _queue_pending_tasks(self, schedule_snapshot):
pending_queue = TaskQueue()
for entry in schedule_snapshot:
task_time = self._take_pending_task_time(entry)
self._cancel_if_completed(entry)
if task_time is None:
continue
task_id = entry.get_next_task_id()
entry.save(update_fields=("next_task_id",))
pri = entry.priority
action = entry.action
pending_queue.enter(task_time, pri, action, entry.name, task_id)
return pending_queue
def _take_pending_task_time(self, entry):
task_times = entry.take_pending()
entry.save(update_fields=("next_task_time", "is_active"))
if not task_times:
return None
most_recent = self._compress_past_task_times(task_times, entry.name)
return most_recent
@staticmethod
def _compress_past_task_times(past, schedule_entry_name):
npast = len(past)
if npast > 1:
msg = "skipping {} {} tasks with times in the past"
logger.warning(msg.format(npast - 1, schedule_entry_name))
most_recent = past[-1]
return most_recent
def _queue_upcoming_tasks(self, schedule_snapshot):
upcoming_queue = TaskQueue()
upcoming_task_times = self._get_upcoming_task_times(schedule_snapshot)
for entry, task_times in upcoming_task_times.items():
task_id = None
pri = entry.priority
action = entry.action
for t in task_times:
upcoming_queue.enter(t, pri, action, entry.name, task_id)
return upcoming_queue
def _get_upcoming_task_times(self, schedule_snapshot):
upcoming_task_times = {}
now = self.timefn()
min_interval = self._get_min_interval(schedule_snapshot)
lookahead = now + min_interval * self.interval_multiplier
for entry in schedule_snapshot:
task_times = entry.get_remaining_times(until=lookahead)
upcoming_task_times[entry] = task_times
return upcoming_task_times
def _get_min_interval(self, schedule_snapshot):
intervals = [e.interval for e in schedule_snapshot if e.interval]
return min(intervals, default=1)
def _cancel_if_completed(self, entry):
if not entry.has_remaining_times():
msg = "no times remaining in {}, removing".format(entry.name)
logger.debug(msg)
self.cancel(entry)
@property
def status(self):
if self.is_alive():
return "running" if self.running else "idle"
return "dead"
def __repr__(self):
s = "running" if self.running else "stopped"
return "<{} status={}>".format(self.__class__.__name__, s)