async def _run_monitor(self, monitor_id: int) -> None:
monitor = self.monitors.get(monitor_id)
if not monitor:
log.debug('Skipping scheduled job for missing monitor %s' %
monitor_id)
return None
monitor.scheduled_job = None
if self.num_running_jobs > self.max_concurrent_jobs:
log.msg('Deferred monitor %s due to to many running jobs' %
monitor)
self.schedule_monitor(monitor, random.randint(10, 30))
stats.inc('jobs_deferred', 'ACT_MON')
return None
self.num_running_jobs += 1
stats.inc('total_jobs_run', 'ACT_MON')
stats.inc('cur_running_jobs', 'ACT_MON')
try:
await monitor.run()
except Exception as e:
stats.dec('cur_running_jobs', 'ACT_MON')
self.num_running_jobs -= 1
log.msg('Monitor run raised error: %s' % (str(e)))
if not monitor.scheduled_job:
self.schedule_monitor(monitor, DEFAULT_MONITOR_INTERVAL)
raise
self.num_running_jobs -= 1
stats.dec('cur_running_jobs', 'ACT_MON')
def __init__(self,
dbcon: DBConnection,
notification_manager: NotificationManager,
max_concurrent_jobs: int,
*,
debug_mode: bool = False,
loop: asyncio.AbstractEventLoop = None) -> None:
self.loop = loop or asyncio.get_event_loop()
self.dbcon = dbcon
self.notification_manager = notification_manager
self.max_concurrent_jobs = max_concurrent_jobs
self.debug_mode = debug_mode
if debug_mode:
log.debug(
'Debug mode active, all monitors will be started immediately')
self.monitor_defs = {} # type: Dict[int, ActiveMonitorDef]
self.monitors = {} # type: Dict[int, ActiveMonitor]
self.num_running_jobs = 0
stats.set('total_jobs_run', 0, 'ACT_MON')
stats.set('cur_running_jobs', 0, 'ACT_MON')
stats.set('num_monitors', 0, 'ACT_MON')
stats.set('jobs_deferred', 0, 'ACT_MON')
stats.set('checks_up', 0, 'ACT_MON')
stats.set('checks_down', 0, 'ACT_MON')
stats.set('checks_unknown', 0, 'ACT_MON')
def main() -> None:
"""Do all setup that doesn't require the event loop, then start it."""
args = parse_cmdline()
config = parse_configfile(args.config)
debug_mode = args.debug or config.getboolean(
'DEFAULT', 'debug', fallback=False)
log.configure_logging(config.get('DEFAULT', 'logtype', fallback='stdout'),
config.get('DEFAULT', 'logfile', fallback=''),
debug_mode)
if debug_mode:
log.debug('Debug mode enabled')
dbcon = sql.DBConnection(config['DATABASE']['host'],
config['DATABASE']['username'],
config['DATABASE']['password'],
config['DATABASE']['dbname'])
notification_cfg = None
if config.has_section('NOTIFICATIONS'):
notification_cfg = config['NOTIFICATIONS']
notification_manager = NotificationManager(notification_cfg, )
active_monitor_manager = ActiveMonitorManager(
dbcon,
notification_manager,
int(
config.get('ACTIVE-MONITORS',
'max-concurrent-jobs',
fallback='200')),
debug_mode=debug_mode,
)
loop = asyncio.get_event_loop()
loop.run_until_complete(
mainloop(loop, config, dbcon, active_monitor_manager))
loop.close()
async def events_websocket_handler(request: web.Request) -> Any:
"""GET view for events websocket.
All the work is done in the WSEventProxy class.
"""
proxy = ws_event_proxy.WSEventProxy(request)
log.debug('Starting event websocket session')
await proxy.run()
log.debug('Ending event websocket session')
return proxy.ws
def decode_plugin_output(output: Union[str, bytes]) -> str:
"""Decode nagios output from latin-1."""
try:
if type(output) == bytes:
output = cast(bytes, output)
output = output.decode('latin-1', 'replace')
except Exception as e:
log.debug('nagios.encode_monitor_output: error: %s' % str(e))
output = ''
return cast(str, output)
def parse_settings(config: Any) -> Optional[Dict[str, Any]]:
ret = {
'url': config.get('http-url'),
} # type: Any
if not ret['url']:
log.debug('HTTP settings missing, no slack notifications will be sent',
'NOTIFICATIONS')
ret = None
else:
log.debug('Valid HTTP notification settings found', 'NOTIFICATIONS')
return ret
def check_missing_schedules(self) -> None:
"""Failsafe to check that no monitors are missing scheduled checks.
This will detect monitors that are lacking missing scheduled jobs.
This shouldn't happen, this is a failsafe in case somthing is buggy.
"""
log.debug('Running monitor missing schedule check')
self.loop.call_later(600, self.check_missing_schedules)
for monitor in self.monitors.values():
if not monitor.deleted and not monitor.monitoring and not monitor.scheduled_job:
log.msg(
'%s is missing scheduled job, this is probably a bug, scheduling now'
% monitor)
self.schedule_monitor(monitor, DEFAULT_MONITOR_INTERVAL)
def schedule_monitor(self, monitor: 'ActiveMonitor',
interval: int) -> None:
log.debug('Scheduling %s for %ds' % (monitor, interval))
if monitor.scheduled_job:
try:
monitor.scheduled_job.cancel()
except ValueError:
pass
monitor.scheduled_job = self.loop.call_later(interval,
self.run_monitor,
monitor.id)
monitor.scheduled_job_ts = time.time() + interval
event.running('SCHEDULE_ACTIVE_MONITOR',
monitor=monitor,
interval=interval)
def parse_settings(config: Any) -> Optional[Dict[str, Any]]:
ret = {
'webhook-url': config.get('slack-webhook-url'),
'tmpl-msg': config.get('slack-tmpl-msg'),
'tmpl-duration': config.get('slack-tmpl-duration', fallback=''),
'tmpl-url': config.get('slack-tmpl-url', fallback='')
} # type: Any
if not ret['webhook-url'] or not ret['tmpl-msg']:
log.debug('Slack settings missing, no slack notifications will be sent', 'NOTIFICATIONS')
ret = None
else:
log.debug('Valid slack notification settings found', 'NOTIFICATIONS')
ret['tmpl-msg'] = jinja2.Template(ret['tmpl-msg'])
if ret['tmpl-duration']:
ret['tmpl-duration'] = jinja2.Template(ret['tmpl-duration'])
if ret['tmpl-url']:
ret['tmpl-url'] = jinja2.Template(ret['tmpl-url'])
return ret
def parse_settings(config: Any) -> Optional[Dict[str, Any]]:
ret = {
'sender': config.get('email-sender'),
'tmpl-subject': config.get('email-tmpl-subject'),
'tmpl-body': config.get('email-tmpl-body'),
'server': config.get('email-server', fallback='localhost')
} # type: Any
if not ret['sender'] or not ret['tmpl-subject'] or not [
'tmpl-body'
] or not ['server']:
log.msg('Email settings missing, no email notifications will be sent',
'NOTIFICATIONS')
ret = None
else:
log.debug('Valid email notification settings found', 'NOTIFICATIONS')
ret['tmpl-subject'] = jinja2.Template(ret['tmpl-subject'])
ret['tmpl-body'] = jinja2.Template(ret['tmpl-body'])
return ret
def parse_settings(config: Any) -> Optional[Dict[str, Any]]:
"""Parse clicksend sms settings.
Should only be called from sms.parse_settings.
"""
ret = {
'provider': 'clicksend',
'username': config.get('sms-clicksend-username'),
'api-key': config.get('sms-clicksend-api-key'),
'sender': config.get('sms-clicksend-sender'),
'tmpl': config.get('sms-tmpl'),
} # type: Any
if not ret['username'] or not ret['api-key'] or not ['sender'] or not ['tmpl']:
log.msg('SMS settings missing, no sms notifications will be sent', 'NOTIFICATIONS')
ret = None
else:
log.debug('Valid SMS notification settings found', 'NOTIFICATIONS')
ret['tmpl'] = jinja2.Template(ret['tmpl'])
return ret