def setup(self, loop=None):
if loop is None:
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
self.loop = loop
self.message_queue = ZeroMQQueue(
settings.ZERO_MQ_HOST,
settings.ZERO_MQ_PORT
)
self.message_queue.connect()
self.scheduler.add_listener(
self.event_listener,
EVENT_JOB_ERROR | EVENT_JOB_MISSED | EVENT_JOB_MAX_INSTANCES
)
class AlamoScheduler(object):
message_queue = None
loop = handler = None
def __init__(self, loop=None):
kw = dict()
if loop:
kw['event_loop'] = loop
self.scheduler = AsyncIOScheduler(**kw)
def setup(self, loop=None):
if loop is None:
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
self.loop = loop
self.message_queue = ZeroMQQueue(
settings.ZERO_MQ_HOST,
settings.ZERO_MQ_PORT
)
self.message_queue.connect()
self.scheduler.add_listener(
self.event_listener,
EVENT_JOB_ERROR | EVENT_JOB_MISSED | EVENT_JOB_MAX_INSTANCES
)
@aiostats.increment()
def _schedule_check(self, check):
"""Schedule check."""
logger.info(
'Check `%s:%s` scheduled!', check['uuid'], check['name']
)
check['scheduled_time'] = datetime.now(tz=pytz_utc).isoformat()
self.message_queue.send(check)
def remove_job(self, job_id):
"""Remove job."""
try:
logger.info('Removing job for check id=`%s`', job_id)
self.scheduler.remove_job(str(job_id))
except JobLookupError:
pass
def schedule_check(self, check):
"""Schedule check with proper interval based on `frequency`.
:param dict check: Check definition
"""
try:
frequency = check['fields']['frequency'] = int(
check['fields']['frequency']
)
logger.info(
'Scheduling check `%s` with id `%s` and interval `%s`',
check['name'], check['id'], frequency
)
jitter = random.randint(0, frequency)
first_run = datetime.now() + timedelta(seconds=jitter)
kw = dict(
seconds=frequency,
id=str(check['uuid']),
next_run_time=first_run,
args=(check,)
)
self.schedule_job(self._schedule_check, **kw)
except KeyError as e:
logger.exception('Failed to schedule check: %s. Exception: %s',
check, e)
def schedule_job(self, method, **kwargs):
"""Add new job to scheduler.
:param method: reference to method that should be scheduled
:param kwargs: additional kwargs passed to `add_job` method
"""
try:
self.scheduler.add_job(
method, 'interval',
misfire_grace_time=settings.JOBS_MISFIRE_GRACE_TIME,
max_instances=settings.JOBS_MAX_INSTANCES,
coalesce=settings.JOBS_COALESCE,
**kwargs
)
except ConflictingIdError as e:
logger.error(e)
def event_listener(self, event):
"""React on events from scheduler.
:param apscheduler.events.JobExecutionEvent event: job execution event
"""
if event.code == EVENT_JOB_MISSED:
aiostats.increment.incr('job.missed')
logger.warning("Job %s scheduler for %s missed.", event.job_id,
event.scheduled_run_time)
elif event.code == EVENT_JOB_ERROR:
aiostats.increment.incr('job.error')
logger.error("Job %s scheduled for %s failed. Exc: %s",
event.job_id,
event.scheduled_run_time,
event.exception)
elif event.code == EVENT_JOB_MAX_INSTANCES:
aiostats.increment.incr('job.max_instances')
logger.warning(
'Job `%s` could not be submitted. '
'Maximum number of running instances was reached.',
event.job_id
)
@aiostats.increment()
def get_jobs(self):
return [job.id for job in self.scheduler.get_jobs()]
async def checks(self, request=None):
uuid = request.match_info.get('uuid', None)
if uuid is None:
jobs = self.get_jobs()
return json_response(data=dict(count=len(jobs), results=jobs))
job = self.scheduler.get_job(uuid)
if job is None:
return json_response(
data={'detail': 'Check does not exists.'}, status=404
)
check, = job.args
return json_response(data=check)
@aiostats.timer()
async def update(self, request=None):
check = await request.json()
check_uuid = check.get('uuid')
check_id = check.get('id')
message = dict(status='ok')
if not check_id or not check_uuid:
return json_response(status=400)
if check_id % settings.SCHEDULER_COUNT != settings.SCHEDULER_NR:
return json_response(data=message, status=202)
job = self.scheduler.get_job(str(check_uuid))
if job:
scheduled_check, = job.args
timestamp = scheduled_check.get('timestamp', 0)
if timestamp > check['timestamp']:
return json_response(data=message, status=202)
message = dict(status='deleted')
self.remove_job(check_uuid)
if any([trigger['enabled'] for trigger in check['triggers']]):
self.schedule_check(check)
message = dict(status='scheduled')
return json_response(data=message, status=202)
def wait_and_kill(self, sig):
logger.warning('Got `%s` signal. Preparing scheduler to exit ...', sig)
self.scheduler.shutdown()
self.loop.stop()
def register_exit_signals(self):
for sig in ['SIGQUIT', 'SIGINT', 'SIGTERM']:
logger.info('Registering handler for `%s` signal '
'in current event loop ...', sig)
self.loop.add_signal_handler(
getattr(signal, sig),
self.wait_and_kill, sig
)
def start(self, loop=None):
"""Start scheduler."""
self.setup(loop=loop)
self.register_exit_signals()
self.scheduler.start()
logger.info(
'Press Ctrl+%s to exit.', 'Break' if os.name == 'nt' else 'C'
)
try:
self.loop.run_forever()
except KeyboardInterrupt:
pass
logger.info('Scheduler was stopped!')