async def _create_db(self) -> None:
# Yes yes, this risks sql injection, but the dbname is from the
# irisett config file, so if you want to sql inject yourself,
# go ahead.
log.msg('Creating missing database %s' % self.dbname)
q = """CREATE DATABASE %s""" % self.dbname
await self.operation(q)
async def middleware_handler(request: web.Request) -> web.Response:
errcode = None
errmsg = None
ret = None
headers = {}
try:
ret = await handler(request)
except errors.NotFound as e:
errcode = 404
errmsg = str(e) or 'not found'
except errors.PermissionDenied as e:
errcode = 401
errmsg = str(e) or 'permission denied'
except errors.MissingLogin as e:
errcode = 401
errmsg = str(e) or 'permission denied'
headers['WWW-Authenticate'] = 'Basic realm="Restricted"'
except errors.InvalidData as e:
errcode = 400
errmsg = str(e) or 'invalid data'
except errors.WebMgmtError as e:
errcode = 400
errmsg = str(e) or 'web error'
except IrisettError as e:
errcode = 400
errmsg = str(e) or 'irisett error'
if errcode:
log.msg(
'Request returning error(%d/%s): %s' %
(errcode, errmsg, request), 'WEBMGMT')
ret = web.Response(status=errcode, text=errmsg, headers=headers)
return ret
async def initialize(self, *, only_init_tables: bool=False):
"""Initialize the DBConnection.
Creates a connection pool using aiomysql and initializes the database
if necessary.
"""
self.pool = await aiomysql.create_pool(
host=self.host, user=self.user, password=self.passwd,
loop=self.loop)
db_exists = await self._check_db_exists()
if not db_exists:
await self._create_db()
db_initialized = False
else:
db_initialized = await self._check_db_initialized()
# We close the pool and create a new one because aiomysql doesn't
# provide an easy way to change the active database for an entire
# pool, just individual connections.
self.pool.terminate()
self.pool = await aiomysql.create_pool(
host=self.host, user=self.user, password=self.passwd,
db=self.dbname, loop=self.loop)
if not db_initialized:
await self._init_db(only_init_tables)
await self._upgrade_db()
log.msg('Database initialized')
async def middleware_handler(request: web.Request) -> web.Response:
errcode = None
errmsg = None
ret = None
try:
ret = await handler(request)
except errors.NotFound as e:
errcode = 404
errmsg = str(e) or 'not found'
except errors.PermissionDenied as e:
errcode = 401
errmsg = str(e) or 'permission denied'
except errors.InvalidData as e:
errcode = 400
errmsg = str(e) or 'invalid data'
except errors.WebAPIError as e:
errcode = 400
errmsg = str(e) or 'api error'
except IrisettError as e:
errcode = 400
errmsg = str(e) or 'irisett error'
if errcode:
log.msg(
'Request returning error(%d/%s): %s' %
(errcode, errmsg, request), 'WEBAPI')
ret = web.Response(status=errcode, text=errmsg)
return ret
async def mainloop(loop: asyncio.AbstractEventLoop,
config: configparser.ConfigParser, dbcon: sql.DBConnection,
active_monitor_manager: ActiveMonitorManager):
"""Perform all setup that requires the event loop and then just wait around forever."""
await dbcon.initialize()
await active_monitor_manager.initialize()
webapi.initialize(
loop,
int(config.get('WEBAPI', 'port', fallback='10000')),
config.get('WEBAPI', 'username'),
config.get('WEBAPI', 'password'),
dbcon,
active_monitor_manager,
)
if config.has_section('WEBMGMT'):
from irisett.webmgmt import webmgmt
webmgmt.initialize(
loop,
int(config.get('WEBMGMT', 'port', fallback='11000')),
config.get('WEBMGMT', 'username'),
config.get('WEBMGMT', 'password'),
dbcon,
active_monitor_manager,
)
active_monitor_manager.start()
stats.set('global_startup', time.time())
log.msg('Irisett startup complete')
while True:
await asyncio.sleep(10)
async def _run_monitor(self, monitor_id: int) -> None:
monitor = self.monitors.get(monitor_id)
if not monitor:
log.debug('Skipping scheduled job for missing monitor %s' %
monitor_id)
return None
monitor.scheduled_job = None
if self.num_running_jobs > self.max_concurrent_jobs:
log.msg('Deferred monitor %s due to to many running jobs' %
monitor)
self.schedule_monitor(monitor, random.randint(10, 30))
stats.inc('jobs_deferred', 'ACT_MON')
return None
self.num_running_jobs += 1
stats.inc('total_jobs_run', 'ACT_MON')
stats.inc('cur_running_jobs', 'ACT_MON')
try:
await monitor.run()
except Exception as e:
stats.dec('cur_running_jobs', 'ACT_MON')
self.num_running_jobs -= 1
log.msg('Monitor run raised error: %s' % (str(e)))
if not monitor.scheduled_job:
self.schedule_monitor(monitor, DEFAULT_MONITOR_INTERVAL)
raise
self.num_running_jobs -= 1
stats.dec('cur_running_jobs', 'ACT_MON')
async def create_active_monitor(
manager: ActiveMonitorManager, args: Dict[str, str],
monitor_def: ActiveMonitorDef) -> ActiveMonitor:
async def _run(cur: sql.Cursor) -> None:
q = """insert into active_monitors (def_id, state, state_ts, msg) values (%s, %s, %s, %s)"""
q_args = (monitor_def.id, 'UNKNOWN', 0, '') # type: Tuple
await cur.execute(q, q_args)
_monitor_id = cur.lastrowid
q = """insert into active_monitor_args (monitor_id, name, value) values (%s, %s, %s)"""
for name, value in args.items():
q_args = (_monitor_id, name, value)
await cur.execute(q, q_args)
return _monitor_id
monitor_def.validate_monitor_args(args)
monitor_id = await manager.dbcon.transact(_run)
monitor = ActiveMonitor(monitor_id,
args,
monitor_def,
'UNKNOWN',
state_ts=0,
msg='',
alert_id=None,
checks_enabled=True,
alerts_enabled=True,
manager=manager)
log.msg('Created active monitor %s' % monitor)
manager.add_monitor(monitor)
return monitor
def initialize(loop: asyncio.AbstractEventLoop, port: int, username: str, password: str, dbcon: DBConnection,
active_monitor_manager: ActiveMonitorManager) -> None:
"""Initialize the webmgmt listener."""
stats.set('num_calls', 0, 'WEBMGMT')
app = web.Application(loop=loop, logger=log.logger,
middlewares=[
middleware.logging_middleware_factory,
middleware.error_handler_middleware_factory,
middleware.basic_auth_middleware_factory,
])
app['username'] = username
app['password'] = password
app['dbcon'] = dbcon
app['active_monitor_manager'] = active_monitor_manager
setup_routes(app)
aiohttp_jinja2.setup(
app,
loader=jinja2.PackageLoader('irisett.webmgmt', 'templates'),
filters={
'timestamp': jinja_filters.timestamp
},
)
listener = loop.create_server(app.make_handler(), '0.0.0.0', port)
asyncio.ensure_future(listener)
log.msg('Webmgmt listening on port %s' % port)
async def _init_db(self, only_init_tables: bool) -> None:
log.msg('Initializing empty database')
commands = sql_data.SQL_ALL
if only_init_tables:
commands = sql_data.SQL_BARE
for command in commands:
await self.operation(command)
async def send_email(loop: asyncio.AbstractEventLoop,
mail_from: str,
mail_to: Union[Iterable, str],
subject: str,
body: str,
server: str = 'localhost') -> None:
"""Send an email to one or more recipients.
Only supports plain text emails with a single message body.
No attachments etc.
"""
if type(mail_to) == str:
mail_to = [mail_to]
smtp = aiosmtplib.SMTP(hostname=server, port=25, loop=loop)
try:
await smtp.connect()
for rcpt in mail_to:
msg = MIMEText(body)
msg['Subject'] = subject
msg['From'] = mail_from
msg['To'] = rcpt
await smtp.send_message(msg)
await smtp.quit()
except aiosmtplib.errors.SMTPException as e:
log.msg('Error sending smtp notification: %s' % (str(e)),
'NOTIFICATIONS')
async def remove_deleted_monitors(dbcon: DBConnection) -> None:
"""Remove any monitors that have previously been set as deleted.
This runs once every time the server starts up.
"""
log.msg('Purging all deleted active monitor')
q = """select id from active_monitors where deleted=true"""
rows = await dbcon.fetch_all(q)
for monitor_id in rows:
await remove_monitor_from_db(dbcon, monitor_id)
async def initialize(self) -> None:
"""Load all data required for the managed main loop to run.
This can't be called from __init__ as it is an async call.
"""
await remove_deleted_monitors(self.dbcon)
self.monitor_defs = await load_monitor_defs(self)
self.monitors = await load_monitors(self)
log.msg('Loaded %d active monitor definitions' %
(len(self.monitor_defs)))
log.msg('Loaded %d active monitors' % (len(self.monitors)))
async def send_slack_notification(url: str, attachments: List[Dict]):
data = {
'attachments': attachments
}
try:
async with aiohttp.ClientSession() as session:
async with session.post(url, data=json.dumps(data), timeout=30) as resp:
if resp.status != 200:
log.msg('Error sending slack notification: http status %s' % (str(resp.status)),
'NOTIFICATION')
except aiohttp.ClientError as e:
log.msg('Error sending slack notification: %s' % (str(e)), 'NOTIFICATIONS')
async def send_http_notification(url: str, in_data: Any):
out_data = json.dumps(in_data)
try:
async with aiohttp.ClientSession() as session:
async with session.post(url, data=out_data, timeout=10) as resp:
if resp.status != 200:
log.msg(
'Error sending http notification: http status %s' %
(str(resp.status)), 'NOTIFICATION')
except aiohttp.ClientError as e:
log.msg('Error sending http notification: %s' % (str(e)),
'NOTIFICATIONS')
async def _upgrade_db(self) -> None:
"""Upgrade to a newer database version if required.
Loops through the commands in sql_data.SQL_UPGRADES and runs them.
"""
cur_version = await self._get_db_version()
for n in range(cur_version + 1, sql_data.CUR_VERSION + 1):
log.msg('Upgrading database to version %d' % n)
if n in sql_data.SQL_UPGRADES:
for command in sql_data.SQL_UPGRADES[n]:
await self.operation(command)
if cur_version != sql_data.CUR_VERSION:
await self._set_db_version(sql_data.CUR_VERSION)
def check_missing_schedules(self) -> None:
"""Failsafe to check that no monitors are missing scheduled checks.
This will detect monitors that are lacking missing scheduled jobs.
This shouldn't happen, this is a failsafe in case somthing is buggy.
"""
log.debug('Running monitor missing schedule check')
self.loop.call_later(600, self.check_missing_schedules)
for monitor in self.monitors.values():
if not monitor.deleted and not monitor.monitoring and not monitor.scheduled_job:
log.msg(
'%s is missing scheduled job, this is probably a bug, scheduling now'
% monitor)
self.schedule_monitor(monitor, DEFAULT_MONITOR_INTERVAL)
def parse_settings(config: Any) -> Optional[Dict[str, Any]]:
provider = config.get('sms-provider')
if not provider:
log.msg('No SMS provider specified, no sms notifications will be sent',
'NOTIFICATIONS')
return None
if provider not in ['clicksend']:
log.msg(
'Unknown SMS provider specified, no sms notifications will be sent',
'NOTIFICATIONS')
return None
ret = None
if provider == 'clicksend':
ret = clicksend.parse_settings(config)
return ret
async def create_active_monitor_def(manager: ActiveMonitorManager, name: str,
description: str, active: bool,
cmdline_filename: str,
cmdline_args_tmpl: str,
description_tmpl: str) -> ActiveMonitorDef:
q = """insert into active_monitor_defs
(name, description, active, cmdline_filename, cmdline_args_tmpl, description_tmpl)
values (%s, %s, %s, %s, %s, %s)"""
q_args = (name, description, active, cmdline_filename, cmdline_args_tmpl,
description_tmpl)
monitor_def_id = await manager.dbcon.operation(q, q_args)
monitor_def = ActiveMonitorDef(monitor_def_id, name, active,
cmdline_filename, cmdline_args_tmpl,
description_tmpl, [], manager)
log.msg('Created active monitor def %s' % monitor_def)
manager.monitor_defs[monitor_def.id] = monitor_def
return monitor_def
def __init__(self,
config: Any,
*,
loop: asyncio.AbstractEventLoop = None) -> None:
self.loop = loop or asyncio.get_event_loop()
if not config:
log.msg(
'Missing config section, no alert notification will be sent',
'NOTIFICATIONS')
self.http_settings = None
self.email_settings = None
self.sms_settings = None
self.slack_settings = None
else:
self.http_settings = http.parse_settings(config)
self.email_settings = email.parse_settings(config)
self.sms_settings = sms.parse_settings(config)
self.slack_settings = slack.parse_settings(config)
def initialize(loop: asyncio.AbstractEventLoop, port: int, username: str, password: str, dbcon: DBConnection,
active_monitor_manager: ActiveMonitorManager) -> None:
"""Initialize the webapi listener."""
stats.set('num_calls', 0, 'WEBAPI')
app = web.Application(loop=loop, logger=log.logger,
middlewares=[
middleware.logging_middleware_factory,
middleware.error_handler_middleware_factory,
middleware.basic_auth_middleware_factory,
])
app['username'] = username
app['password'] = password
app['dbcon'] = dbcon
app['active_monitor_manager'] = active_monitor_manager
setup_routes(app)
listener = loop.create_server(app.make_handler(), '0.0.0.0', port)
loop.create_task(listener)
log.msg('Webapi listening on port %s' % port)
def running(self, event_name: str, **kwargs: Any) -> None:
"""An event is running.
Listener callbacks will be called with:
callback(listener-dict, event-name, timestamp, arg-dict)
"""
stats.inc('events_fired', 'EVENT')
if not self.listeners:
return
timestamp = time.time()
for listener in self.listeners:
if not listener.wants_event(event_name, kwargs):
continue
try:
t = listener.callback(listener, event_name, timestamp, kwargs)
asyncio.ensure_future(t)
except Exception as e:
log.msg('Failed to run event listener callback: %s' % str(e))
def parse_settings(config: Any) -> Optional[Dict[str, Any]]:
ret = {
'sender': config.get('email-sender'),
'tmpl-subject': config.get('email-tmpl-subject'),
'tmpl-body': config.get('email-tmpl-body'),
'server': config.get('email-server', fallback='localhost')
} # type: Any
if not ret['sender'] or not ret['tmpl-subject'] or not [
'tmpl-body'
] or not ['server']:
log.msg('Email settings missing, no email notifications will be sent',
'NOTIFICATIONS')
ret = None
else:
log.debug('Valid email notification settings found', 'NOTIFICATIONS')
ret['tmpl-subject'] = jinja2.Template(ret['tmpl-subject'])
ret['tmpl-body'] = jinja2.Template(ret['tmpl-body'])
return ret
def parse_settings(config: Any) -> Optional[Dict[str, Any]]:
"""Parse clicksend sms settings.
Should only be called from sms.parse_settings.
"""
ret = {
'provider': 'clicksend',
'username': config.get('sms-clicksend-username'),
'api-key': config.get('sms-clicksend-api-key'),
'sender': config.get('sms-clicksend-sender'),
'tmpl': config.get('sms-tmpl'),
} # type: Any
if not ret['username'] or not ret['api-key'] or not ['sender'] or not ['tmpl']:
log.msg('SMS settings missing, no sms notifications will be sent', 'NOTIFICATIONS')
ret = None
else:
log.debug('Valid SMS notification settings found', 'NOTIFICATIONS')
ret['tmpl'] = jinja2.Template(ret['tmpl'])
return ret
async def send_sms(recipients: Iterable[str], msg: str, username: str, api_key: str, sender: str):
data = {
'messages': [],
} # type: Dict[str, List]
for recipient in recipients:
data['messages'].append({
'source': 'python',
'from': sender,
'body': msg[:140],
'to': recipient,
'schedule': ''
})
try:
async with aiohttp.ClientSession(headers={'Content-Type': 'application/json'},
auth=aiohttp.BasicAuth(username, api_key)) as session:
async with session.post(CLICKSEND_URL, data=json.dumps(data), timeout=30) as resp:
if resp.status != 200:
log.msg('Error sending clicksend sms notification: http status %s' % (str(resp.status)),
'NOTIFICATION')
except aiohttp.ClientError as e:
log.msg('Error sending clicksend sms notification: %s' % (str(e)), 'NOTIFICATIONS')
async def middleware_handler(request: web.Request) -> web.Response:
ok = False
auth_token = request.headers.get('Authorization')
if auth_token and auth_token.startswith('Basic '):
auth_token = auth_token[6:]
try:
auth_bytes = base64.b64decode(
auth_token) # type: Optional[bytes]
except binascii.Error:
auth_bytes = None
if auth_bytes:
auth_str = auth_bytes.decode('utf-8', errors='ignore')
if ':' in auth_str:
username, password = auth_str.split(':', 1)
if username == app['username'] and password == app[
'password']:
ok = True
if not ok:
log.msg('Unauthorized request: %s' % request, 'WEBMGMT')
raise errors.MissingLogin('Unauthorized')
return await handler(request)
async def middleware_handler(request: web.Request) -> web.Response:
stats.inc('num_calls', 'WEBMGMT')
log.msg('Received request: %s' % request, 'WEBMGMT')
return await handler(request)
