async def _create_db(self) -> None: # Yes yes, this risks sql injection, but the dbname is from the # irisett config file, so if you want to sql inject yourself, # go ahead. log.msg('Creating missing database %s' % self.dbname) q = """CREATE DATABASE %s""" % self.dbname await self.operation(q)
async def middleware_handler(request: web.Request) -> web.Response: errcode = None errmsg = None ret = None headers = {} try: ret = await handler(request) except errors.NotFound as e: errcode = 404 errmsg = str(e) or 'not found' except errors.PermissionDenied as e: errcode = 401 errmsg = str(e) or 'permission denied' except errors.MissingLogin as e: errcode = 401 errmsg = str(e) or 'permission denied' headers['WWW-Authenticate'] = 'Basic realm="Restricted"' except errors.InvalidData as e: errcode = 400 errmsg = str(e) or 'invalid data' except errors.WebMgmtError as e: errcode = 400 errmsg = str(e) or 'web error' except IrisettError as e: errcode = 400 errmsg = str(e) or 'irisett error' if errcode: log.msg( 'Request returning error(%d/%s): %s' % (errcode, errmsg, request), 'WEBMGMT') ret = web.Response(status=errcode, text=errmsg, headers=headers) return ret
async def initialize(self, *, only_init_tables: bool=False): """Initialize the DBConnection. Creates a connection pool using aiomysql and initializes the database if necessary. """ self.pool = await aiomysql.create_pool( host=self.host, user=self.user, password=self.passwd, loop=self.loop) db_exists = await self._check_db_exists() if not db_exists: await self._create_db() db_initialized = False else: db_initialized = await self._check_db_initialized() # We close the pool and create a new one because aiomysql doesn't # provide an easy way to change the active database for an entire # pool, just individual connections. self.pool.terminate() self.pool = await aiomysql.create_pool( host=self.host, user=self.user, password=self.passwd, db=self.dbname, loop=self.loop) if not db_initialized: await self._init_db(only_init_tables) await self._upgrade_db() log.msg('Database initialized')
async def middleware_handler(request: web.Request) -> web.Response: errcode = None errmsg = None ret = None try: ret = await handler(request) except errors.NotFound as e: errcode = 404 errmsg = str(e) or 'not found' except errors.PermissionDenied as e: errcode = 401 errmsg = str(e) or 'permission denied' except errors.InvalidData as e: errcode = 400 errmsg = str(e) or 'invalid data' except errors.WebAPIError as e: errcode = 400 errmsg = str(e) or 'api error' except IrisettError as e: errcode = 400 errmsg = str(e) or 'irisett error' if errcode: log.msg( 'Request returning error(%d/%s): %s' % (errcode, errmsg, request), 'WEBAPI') ret = web.Response(status=errcode, text=errmsg) return ret
async def mainloop(loop: asyncio.AbstractEventLoop, config: configparser.ConfigParser, dbcon: sql.DBConnection, active_monitor_manager: ActiveMonitorManager): """Perform all setup that requires the event loop and then just wait around forever.""" await dbcon.initialize() await active_monitor_manager.initialize() webapi.initialize( loop, int(config.get('WEBAPI', 'port', fallback='10000')), config.get('WEBAPI', 'username'), config.get('WEBAPI', 'password'), dbcon, active_monitor_manager, ) if config.has_section('WEBMGMT'): from irisett.webmgmt import webmgmt webmgmt.initialize( loop, int(config.get('WEBMGMT', 'port', fallback='11000')), config.get('WEBMGMT', 'username'), config.get('WEBMGMT', 'password'), dbcon, active_monitor_manager, ) active_monitor_manager.start() stats.set('global_startup', time.time()) log.msg('Irisett startup complete') while True: await asyncio.sleep(10)
async def _run_monitor(self, monitor_id: int) -> None: monitor = self.monitors.get(monitor_id) if not monitor: log.debug('Skipping scheduled job for missing monitor %s' % monitor_id) return None monitor.scheduled_job = None if self.num_running_jobs > self.max_concurrent_jobs: log.msg('Deferred monitor %s due to to many running jobs' % monitor) self.schedule_monitor(monitor, random.randint(10, 30)) stats.inc('jobs_deferred', 'ACT_MON') return None self.num_running_jobs += 1 stats.inc('total_jobs_run', 'ACT_MON') stats.inc('cur_running_jobs', 'ACT_MON') try: await monitor.run() except Exception as e: stats.dec('cur_running_jobs', 'ACT_MON') self.num_running_jobs -= 1 log.msg('Monitor run raised error: %s' % (str(e))) if not monitor.scheduled_job: self.schedule_monitor(monitor, DEFAULT_MONITOR_INTERVAL) raise self.num_running_jobs -= 1 stats.dec('cur_running_jobs', 'ACT_MON')
async def create_active_monitor( manager: ActiveMonitorManager, args: Dict[str, str], monitor_def: ActiveMonitorDef) -> ActiveMonitor: async def _run(cur: sql.Cursor) -> None: q = """insert into active_monitors (def_id, state, state_ts, msg) values (%s, %s, %s, %s)""" q_args = (monitor_def.id, 'UNKNOWN', 0, '') # type: Tuple await cur.execute(q, q_args) _monitor_id = cur.lastrowid q = """insert into active_monitor_args (monitor_id, name, value) values (%s, %s, %s)""" for name, value in args.items(): q_args = (_monitor_id, name, value) await cur.execute(q, q_args) return _monitor_id monitor_def.validate_monitor_args(args) monitor_id = await manager.dbcon.transact(_run) monitor = ActiveMonitor(monitor_id, args, monitor_def, 'UNKNOWN', state_ts=0, msg='', alert_id=None, checks_enabled=True, alerts_enabled=True, manager=manager) log.msg('Created active monitor %s' % monitor) manager.add_monitor(monitor) return monitor
def initialize(loop: asyncio.AbstractEventLoop, port: int, username: str, password: str, dbcon: DBConnection, active_monitor_manager: ActiveMonitorManager) -> None: """Initialize the webmgmt listener.""" stats.set('num_calls', 0, 'WEBMGMT') app = web.Application(loop=loop, logger=log.logger, middlewares=[ middleware.logging_middleware_factory, middleware.error_handler_middleware_factory, middleware.basic_auth_middleware_factory, ]) app['username'] = username app['password'] = password app['dbcon'] = dbcon app['active_monitor_manager'] = active_monitor_manager setup_routes(app) aiohttp_jinja2.setup( app, loader=jinja2.PackageLoader('irisett.webmgmt', 'templates'), filters={ 'timestamp': jinja_filters.timestamp }, ) listener = loop.create_server(app.make_handler(), '0.0.0.0', port) asyncio.ensure_future(listener) log.msg('Webmgmt listening on port %s' % port)
async def _init_db(self, only_init_tables: bool) -> None: log.msg('Initializing empty database') commands = sql_data.SQL_ALL if only_init_tables: commands = sql_data.SQL_BARE for command in commands: await self.operation(command)
async def send_email(loop: asyncio.AbstractEventLoop, mail_from: str, mail_to: Union[Iterable, str], subject: str, body: str, server: str = 'localhost') -> None: """Send an email to one or more recipients. Only supports plain text emails with a single message body. No attachments etc. """ if type(mail_to) == str: mail_to = [mail_to] smtp = aiosmtplib.SMTP(hostname=server, port=25, loop=loop) try: await smtp.connect() for rcpt in mail_to: msg = MIMEText(body) msg['Subject'] = subject msg['From'] = mail_from msg['To'] = rcpt await smtp.send_message(msg) await smtp.quit() except aiosmtplib.errors.SMTPException as e: log.msg('Error sending smtp notification: %s' % (str(e)), 'NOTIFICATIONS')
async def remove_deleted_monitors(dbcon: DBConnection) -> None: """Remove any monitors that have previously been set as deleted. This runs once every time the server starts up. """ log.msg('Purging all deleted active monitor') q = """select id from active_monitors where deleted=true""" rows = await dbcon.fetch_all(q) for monitor_id in rows: await remove_monitor_from_db(dbcon, monitor_id)
async def initialize(self) -> None: """Load all data required for the managed main loop to run. This can't be called from __init__ as it is an async call. """ await remove_deleted_monitors(self.dbcon) self.monitor_defs = await load_monitor_defs(self) self.monitors = await load_monitors(self) log.msg('Loaded %d active monitor definitions' % (len(self.monitor_defs))) log.msg('Loaded %d active monitors' % (len(self.monitors)))
async def send_slack_notification(url: str, attachments: List[Dict]): data = { 'attachments': attachments } try: async with aiohttp.ClientSession() as session: async with session.post(url, data=json.dumps(data), timeout=30) as resp: if resp.status != 200: log.msg('Error sending slack notification: http status %s' % (str(resp.status)), 'NOTIFICATION') except aiohttp.ClientError as e: log.msg('Error sending slack notification: %s' % (str(e)), 'NOTIFICATIONS')
async def send_http_notification(url: str, in_data: Any): out_data = json.dumps(in_data) try: async with aiohttp.ClientSession() as session: async with session.post(url, data=out_data, timeout=10) as resp: if resp.status != 200: log.msg( 'Error sending http notification: http status %s' % (str(resp.status)), 'NOTIFICATION') except aiohttp.ClientError as e: log.msg('Error sending http notification: %s' % (str(e)), 'NOTIFICATIONS')
async def _upgrade_db(self) -> None: """Upgrade to a newer database version if required. Loops through the commands in sql_data.SQL_UPGRADES and runs them. """ cur_version = await self._get_db_version() for n in range(cur_version + 1, sql_data.CUR_VERSION + 1): log.msg('Upgrading database to version %d' % n) if n in sql_data.SQL_UPGRADES: for command in sql_data.SQL_UPGRADES[n]: await self.operation(command) if cur_version != sql_data.CUR_VERSION: await self._set_db_version(sql_data.CUR_VERSION)
def check_missing_schedules(self) -> None: """Failsafe to check that no monitors are missing scheduled checks. This will detect monitors that are lacking missing scheduled jobs. This shouldn't happen, this is a failsafe in case somthing is buggy. """ log.debug('Running monitor missing schedule check') self.loop.call_later(600, self.check_missing_schedules) for monitor in self.monitors.values(): if not monitor.deleted and not monitor.monitoring and not monitor.scheduled_job: log.msg( '%s is missing scheduled job, this is probably a bug, scheduling now' % monitor) self.schedule_monitor(monitor, DEFAULT_MONITOR_INTERVAL)
def parse_settings(config: Any) -> Optional[Dict[str, Any]]: provider = config.get('sms-provider') if not provider: log.msg('No SMS provider specified, no sms notifications will be sent', 'NOTIFICATIONS') return None if provider not in ['clicksend']: log.msg( 'Unknown SMS provider specified, no sms notifications will be sent', 'NOTIFICATIONS') return None ret = None if provider == 'clicksend': ret = clicksend.parse_settings(config) return ret
async def create_active_monitor_def(manager: ActiveMonitorManager, name: str, description: str, active: bool, cmdline_filename: str, cmdline_args_tmpl: str, description_tmpl: str) -> ActiveMonitorDef: q = """insert into active_monitor_defs (name, description, active, cmdline_filename, cmdline_args_tmpl, description_tmpl) values (%s, %s, %s, %s, %s, %s)""" q_args = (name, description, active, cmdline_filename, cmdline_args_tmpl, description_tmpl) monitor_def_id = await manager.dbcon.operation(q, q_args) monitor_def = ActiveMonitorDef(monitor_def_id, name, active, cmdline_filename, cmdline_args_tmpl, description_tmpl, [], manager) log.msg('Created active monitor def %s' % monitor_def) manager.monitor_defs[monitor_def.id] = monitor_def return monitor_def
def __init__(self, config: Any, *, loop: asyncio.AbstractEventLoop = None) -> None: self.loop = loop or asyncio.get_event_loop() if not config: log.msg( 'Missing config section, no alert notification will be sent', 'NOTIFICATIONS') self.http_settings = None self.email_settings = None self.sms_settings = None self.slack_settings = None else: self.http_settings = http.parse_settings(config) self.email_settings = email.parse_settings(config) self.sms_settings = sms.parse_settings(config) self.slack_settings = slack.parse_settings(config)
def initialize(loop: asyncio.AbstractEventLoop, port: int, username: str, password: str, dbcon: DBConnection, active_monitor_manager: ActiveMonitorManager) -> None: """Initialize the webapi listener.""" stats.set('num_calls', 0, 'WEBAPI') app = web.Application(loop=loop, logger=log.logger, middlewares=[ middleware.logging_middleware_factory, middleware.error_handler_middleware_factory, middleware.basic_auth_middleware_factory, ]) app['username'] = username app['password'] = password app['dbcon'] = dbcon app['active_monitor_manager'] = active_monitor_manager setup_routes(app) listener = loop.create_server(app.make_handler(), '0.0.0.0', port) loop.create_task(listener) log.msg('Webapi listening on port %s' % port)
def running(self, event_name: str, **kwargs: Any) -> None: """An event is running. Listener callbacks will be called with: callback(listener-dict, event-name, timestamp, arg-dict) """ stats.inc('events_fired', 'EVENT') if not self.listeners: return timestamp = time.time() for listener in self.listeners: if not listener.wants_event(event_name, kwargs): continue try: t = listener.callback(listener, event_name, timestamp, kwargs) asyncio.ensure_future(t) except Exception as e: log.msg('Failed to run event listener callback: %s' % str(e))
def parse_settings(config: Any) -> Optional[Dict[str, Any]]: ret = { 'sender': config.get('email-sender'), 'tmpl-subject': config.get('email-tmpl-subject'), 'tmpl-body': config.get('email-tmpl-body'), 'server': config.get('email-server', fallback='localhost') } # type: Any if not ret['sender'] or not ret['tmpl-subject'] or not [ 'tmpl-body' ] or not ['server']: log.msg('Email settings missing, no email notifications will be sent', 'NOTIFICATIONS') ret = None else: log.debug('Valid email notification settings found', 'NOTIFICATIONS') ret['tmpl-subject'] = jinja2.Template(ret['tmpl-subject']) ret['tmpl-body'] = jinja2.Template(ret['tmpl-body']) return ret
def parse_settings(config: Any) -> Optional[Dict[str, Any]]: """Parse clicksend sms settings. Should only be called from sms.parse_settings. """ ret = { 'provider': 'clicksend', 'username': config.get('sms-clicksend-username'), 'api-key': config.get('sms-clicksend-api-key'), 'sender': config.get('sms-clicksend-sender'), 'tmpl': config.get('sms-tmpl'), } # type: Any if not ret['username'] or not ret['api-key'] or not ['sender'] or not ['tmpl']: log.msg('SMS settings missing, no sms notifications will be sent', 'NOTIFICATIONS') ret = None else: log.debug('Valid SMS notification settings found', 'NOTIFICATIONS') ret['tmpl'] = jinja2.Template(ret['tmpl']) return ret
async def send_sms(recipients: Iterable[str], msg: str, username: str, api_key: str, sender: str): data = { 'messages': [], } # type: Dict[str, List] for recipient in recipients: data['messages'].append({ 'source': 'python', 'from': sender, 'body': msg[:140], 'to': recipient, 'schedule': '' }) try: async with aiohttp.ClientSession(headers={'Content-Type': 'application/json'}, auth=aiohttp.BasicAuth(username, api_key)) as session: async with session.post(CLICKSEND_URL, data=json.dumps(data), timeout=30) as resp: if resp.status != 200: log.msg('Error sending clicksend sms notification: http status %s' % (str(resp.status)), 'NOTIFICATION') except aiohttp.ClientError as e: log.msg('Error sending clicksend sms notification: %s' % (str(e)), 'NOTIFICATIONS')
async def middleware_handler(request: web.Request) -> web.Response: ok = False auth_token = request.headers.get('Authorization') if auth_token and auth_token.startswith('Basic '): auth_token = auth_token[6:] try: auth_bytes = base64.b64decode( auth_token) # type: Optional[bytes] except binascii.Error: auth_bytes = None if auth_bytes: auth_str = auth_bytes.decode('utf-8', errors='ignore') if ':' in auth_str: username, password = auth_str.split(':', 1) if username == app['username'] and password == app[ 'password']: ok = True if not ok: log.msg('Unauthorized request: %s' % request, 'WEBMGMT') raise errors.MissingLogin('Unauthorized') return await handler(request)
async def middleware_handler(request: web.Request) -> web.Response: stats.inc('num_calls', 'WEBMGMT') log.msg('Received request: %s' % request, 'WEBMGMT') return await handler(request)