def on_message(self, body, message):
try:
alert = Alert.parse(body)
alertid = alert.get_id()
except Exception as e:
LOG.warning(e)
return
if alert.repeat:
message.ack()
return
if alert.status not in ['open', 'closed']:
message.ack()
return
if (alert.severity not in ['critical', 'major']
and alert.previous_severity not in ['critical', 'major']):
message.ack()
return
if alertid in on_hold:
if alert.severity in ['normal', 'ok', 'cleared']:
try:
del on_hold[alertid]
except KeyError:
pass
message.ack()
else:
on_hold[alertid] = (alert, time.time() + self.hold_time)
message.ack()
else:
on_hold[alertid] = (alert, time.time() + self.hold_time)
message.ack()
def test_rules_evaluation(alert_spec, input_rules, expected_contacts):
'''
Test that rules are properly evaluated
'''
with patch.dict(mailer.OPTIONS, mailer.DEFAULT_OPTIONS):
mailer.OPTIONS['mail_to'] = []
mailer.OPTIONS['group_rules'] = input_rules
mail_sender = mailer.MailSender()
with patch.object(mail_sender, '_send_email_message') as _sem:
alert = Alert.parse(alert_spec)
_, emailed_contacts = mail_sender.send_email(alert)
assert _sem.call_count == 1
assert emailed_contacts == expected_contacts
def test_rules_evaluation(alert_spec, input_rules, expected_mail_contacts,
expected_telegram_contacts):
'''
Test that rules are properly evaluated
'''
with patch.dict(trigger.OPTIONS, trigger.DEFAULT_OPTIONS):
with patch.dict(trigger.MAIL_OPTIONS, trigger.DEFAULT_MAIL_OPTIONS):
with patch.dict(trigger.TELEGRAM_OPTIONS,
trigger.DEFAULT_TELEGRAM_OPTIONS):
trigger.OPTIONS['group_rules'] = input_rules
mail_sender = trigger.Trigger()
with patch.object(mail_sender, '_send_email_message') as _sem:
alert = Alert.parse(alert_spec)
emailed_contacts, telegram_contacts = mail_sender.diagnose(
alert)
assert _sem.call_count == 1
assert emailed_contacts == expected_mail_contacts
assert telegram_contacts == expected_telegram_contacts
def on_message(self, body, message):
sevs = list(OPTIONS['severities'])
if not sevs:
sevs = ['critical', 'major']
try:
alert = Alert.parse(body)
alertid = alert.get_id()
except Exception as e:
LOG.warn(e)
return
LOG.debug('Alert received from the queue (id: %s)', alertid)
if alert.repeat:
LOG.debug('Ignored alert %s: repeat state', alertid)
message.ack()
return
if alert.status not in ['open', 'closed']:
LOG.debug('Ignored alert %s: not in open or closed state', alertid)
message.ack()
return
if (alert.severity not in sevs
and alert.previous_severity not in sevs):
LOG.debug(
'Ignored alert %s: severity or previous_severity does not matche the severities configuration (%s)',
alertid, sevs)
message.ack()
return
if alertid in on_hold:
if alert.severity in ['normal', 'ok', 'cleared']:
try:
del on_hold[alertid]
except KeyError:
pass
message.ack()
else:
on_hold[alertid] = (alert, time.time() + HOLD_TIME)
message.ack()
else:
on_hold[alertid] = (alert, time.time() + HOLD_TIME)
message.ack()
def on_message(self, body, message):
sevs = list(OPTIONS['severities'])
if not sevs:
sevs = ['critical', 'major']
try:
alert = Alert.parse(body)
alertid = alert.get_id()
except Exception as e:
LOG.warn(e)
return
if alert.repeat:
message.ack()
return
if alert.status not in ['open', 'closed']:
message.ack()
return
if (
alert.severity not in sevs and
alert.previous_severity not in sevs
):
message.ack()
return
if alertid in on_hold:
if alert.severity in ['normal', 'ok', 'cleared']:
try:
del on_hold[alertid]
except KeyError:
pass
message.ack()
else:
on_hold[alertid] = (alert, time.time() + HOLD_TIME)
message.ack()
else:
on_hold[alertid] = (alert, time.time() + HOLD_TIME)
message.ack()
def send_alert(self, resource, event, **kwargs):
data = {
'resource': resource,
'event': event,
'environment': kwargs.get('environment'),
'severity': kwargs.get('severity'),
'correlate': kwargs.get('correlate', None) or list(),
'service': kwargs.get('service', None) or list(),
'group': kwargs.get('group'),
'value': kwargs.get('value'),
'text': kwargs.get('text'),
'tags': kwargs.get('tags', None) or list(),
'attributes': kwargs.get('attributes', None) or dict(),
'origin': kwargs.get('origin'),
'type': kwargs.get('type'),
'createTime': datetime.utcnow(),
'timeout': kwargs.get('timeout'),
'rawData': kwargs.get('raw_data'),
'customer': kwargs.get('customer')
}
r = self.http.post('/alert', data)
alert = Alert.parse(r['alert']) if 'alert' in r else None
return r.get('id', '-'), alert, r.get('message', None)
def send_alert(self, resource, event, **kwargs):
data = {
'resource': resource,
'event': event,
'environment': kwargs.get('environment'),
'severity': kwargs.get('severity'),
'correlate': kwargs.get('correlate', None) or list(),
'service': kwargs.get('service', None) or list(),
'group': kwargs.get('group'),
'value': kwargs.get('value'),
'text': kwargs.get('text'),
'tags': kwargs.get('tags', None) or list(),
'attributes': kwargs.get('attributes', None) or dict(),
'origin': kwargs.get('origin'),
'type': kwargs.get('type'),
'createTime': datetime.utcnow(),
'timeout': kwargs.get('timeout'),
'rawData': kwargs.get('raw_data'),
'customer': kwargs.get('customer')
}
r = self.http.post('/alert', data)
alert = Alert.parse(r['alert']) if 'alert' in r else None
return r.get('id', '-'), alert, r.get('message', None)
def on_message(self, body, message):
try:
alert = Alert.parse(body)
alertid = alert.get_id()
except Exception as e:
LOG.warn(e)
return
if alert.repeat:
message.ack()
return
if alert.status not in ['open', 'closed']:
message.ack()
return
if (
alert.severity not in ['critical', 'major'] and
alert.previous_severity not in ['critical', 'major']
):
message.ack()
return
if alertid in on_hold:
if alert.severity in ['normal', 'ok', 'cleared']:
try:
del on_hold[alertid]
except KeyError:
pass
message.ack()
else:
on_hold[alertid] = (alert, time.time() + HOLD_TIME)
message.ack()
else:
on_hold[alertid] = (alert, time.time() + HOLD_TIME)
message.ack()
def get_alerts(self, query=None, page=1, page_size=100):
r = self.http.get('/alerts', query, page=page, page_size=page_size)
return [Alert.parse(a) for a in r['alerts']]
def get_alert(self, id):
return Alert.parse(self.http.get('/alert/%s' % id)['alert'])
def get_alert(self, id):
return Alert.parse(self.http.get('/alert/%s' % id)['alert'])
def get_alerts(self, query=None):
r = self.http.get('/alerts', query)
return [Alert.parse(a) for a in r['alerts']]
def cli(obj, ids, query, filters, display, from_date=None):
"""Query for alerts based on search filter criteria."""
client = obj['client']
timezone = obj['timezone']
if ids:
query = [('id', x) for x in ids]
elif query:
query = [('q', query)]
else:
query = build_query(filters)
if from_date:
query.append(('from-date', from_date))
r = client.http.get('/alerts', query, page=1, page_size=1000)
if obj['output'] == 'json':
click.echo(json.dumps(r['alerts'], sort_keys=True, indent=4, ensure_ascii=False))
elif obj['output'] in ['json_lines', 'jsonl', 'ndjson']:
for alert in r['alerts']:
click.echo(json.dumps(alert, ensure_ascii=False))
else:
alerts = [Alert.parse(a) for a in r['alerts']]
last_time = r['lastTime']
auto_refresh = r['autoRefresh']
if display == 'oneline':
headers = {'id': 'ID', 'lastReceiveTime': 'LAST RECEIVED', 'severity': 'SEVERITY', 'status': 'STATUS',
'duplicateCount': 'DUPL', 'customer': 'CUSTOMER', 'environment': 'ENVIRONMENT', 'service': 'SERVICE',
'resource': 'RESOURCE', 'group': 'GROUP', 'event': 'EVENT', 'value': 'VALUE', 'text': 'DESCRIPTION'}
data = [{k: v for k, v in a.tabular(timezone).items() if k in headers.keys()} for a in alerts]
click.echo(tabulate(data, headers=headers, tablefmt=obj['output']))
else:
for alert in reversed(alerts):
color = COLOR_MAP.get(alert.severity, {'fg': 'white'})
click.secho('{}|{}|{}|{:5d}|{}|{:<5s}|{:<10s}|{:<18s}|{:12s}|{:16s}|{:12s}'.format(
alert.id[0:8],
DateTime.localtime(alert.last_receive_time, timezone),
alert.severity,
alert.duplicate_count,
alert.customer or '-',
alert.environment,
','.join(alert.service),
alert.resource,
alert.group,
alert.event,
alert.value or 'n/a'), fg=color['fg'])
click.secho(f' |{alert.text}', fg=color['fg'])
if display == 'full':
click.secho(' severity | {} -> {}'.format(alert.previous_severity,
alert.severity), fg=color['fg'])
click.secho(f' trend | {alert.trend_indication}', fg=color['fg'])
click.secho(f' status | {alert.status}', fg=color['fg'])
click.secho(f' resource | {alert.resource}', fg=color['fg'])
click.secho(f' group | {alert.group}', fg=color['fg'])
click.secho(f' event | {alert.event}', fg=color['fg'])
click.secho(f' value | {alert.value}', fg=color['fg'])
click.secho(' tags | {}'.format(' '.join(alert.tags)), fg=color['fg'])
for key, value in alert.attributes.items():
click.secho(f' {key.ljust(10)} | {value}', fg=color['fg'])
latency = alert.receive_time - alert.create_time
click.secho(' time created | {}'.format(
DateTime.localtime(alert.create_time, timezone)), fg=color['fg'])
click.secho(' time received | {}'.format(
DateTime.localtime(alert.receive_time, timezone)), fg=color['fg'])
click.secho(' last received | {}'.format(
DateTime.localtime(alert.last_receive_time, timezone)), fg=color['fg'])
click.secho(f' latency | {latency.microseconds / 1000}ms', fg=color['fg'])
click.secho(f' timeout | {alert.timeout}s', fg=color['fg'])
click.secho(f' alert id | {alert.id}', fg=color['fg'])
click.secho(f' last recv id | {alert.last_receive_id}', fg=color['fg'])
click.secho(f' customer | {alert.customer}', fg=color['fg'])
click.secho(f' environment | {alert.environment}', fg=color['fg'])
click.secho(' service | {}'.format(','.join(alert.service)), fg=color['fg'])
click.secho(f' resource | {alert.resource}', fg=color['fg'])
click.secho(f' type | {alert.event_type}', fg=color['fg'])
click.secho(f' repeat | {alert.repeat}', fg=color['fg'])
click.secho(f' origin | {alert.origin}', fg=color['fg'])
click.secho(' correlate | {}'.format(','.join(alert.correlate)), fg=color['fg'])
return auto_refresh, last_time
def update(self):
self.lines, self.cols = self.screen.getmaxyx()
self.screen.clear()
now = datetime.utcnow()
status = self.client.mgmt_status()
version = status['version']
# draw header
self._addstr(0, 0, self.client.endpoint, curses.A_BOLD)
self._addstr(0, 'C', 'alerta {}'.format(version), curses.A_BOLD)
self._addstr(0, 'R', '{}'.format(now.strftime('%H:%M:%S %d/%m/%y')),
curses.A_BOLD)
# TODO - draw bars
# draw alerts
text_width = self.cols - 95
self._addstr(
2, 1,
'Sev. Time Dupl. Customer Env. Service Resource Group Event'
+ ' Value Text' + ' ' * (text_width - 4),
curses.A_UNDERLINE)
def short_sev(severity):
return self.SEVERITY_MAP.get(severity,
self.SEVERITY_MAP['unknown'])[0]
def color(severity):
return self.SEVERITY_MAP.get(severity,
self.SEVERITY_MAP['unknown'])[1]
r = self.client.http.get('/alerts')
alerts = [Alert.parse(a) for a in r['alerts']]
last_time = DateTime.parse(r['lastTime'])
for i, alert in enumerate(alerts):
row = i + 3
if row >= self.lines - 2: # leave room for footer
break
text = '{:<4} {} {:5d} {:8.8} {:<12} {:<12} {:<12.12} {:5.5} {:<12.12} {:<5.5} {:.{width}}'.format(
short_sev(alert.severity),
DateTime.localtime(alert.last_receive_time,
self.timezone,
fmt='%H:%M:%S'),
alert.duplicate_count,
alert.customer or '-',
alert.environment,
','.join(alert.service),
alert.resource,
alert.group,
alert.event,
alert.value or 'n/a',
alert.text,
width=text_width)
# XXX - needed to support python2 and python3
if not isinstance(text, str):
text = text.encode('ascii', errors='replace')
self._addstr(row, 1, text, color(alert.severity))
# draw footer
self._addstr(self.lines - 1, 0,
'Last Update: {}'.format(last_time.strftime('%H:%M:%S')),
curses.A_BOLD)
self._addstr(
self.lines - 1, 'C',
'{} - {}'.format(r['status'], r.get('message',
'no errors')), curses.A_BOLD)
self._addstr(self.lines - 1, 'R', 'Count: {}'.format(r['total']),
curses.A_BOLD)
self.screen.refresh()
def cli(obj, ids, query, filters, display, from_date=None):
"""Query for alerts based on search filter criteria."""
client = obj['client']
timezone = obj['timezone']
if ids:
query = [('id', x) for x in ids]
elif query:
query = [('q', query)]
else:
query = build_query(filters)
if from_date:
query.append(('from-date', from_date))
r = client.http.get('/alerts', query, page=1, page_size=1000)
if obj['output'] == 'json':
click.echo(json.dumps(r['alerts'], sort_keys=True, indent=4, ensure_ascii=False))
elif obj['output'] in ['json_lines', 'jsonl', 'ndjson']:
for alert in r['alerts']:
click.echo(json.dumps(alert, ensure_ascii=False))
else:
alerts = [Alert.parse(a) for a in r['alerts']]
last_time = r['lastTime']
auto_refresh = r['autoRefresh']
if display == 'tabular':
headers = {'id': 'ID', 'lastReceiveTime': 'LAST RECEIVED', 'severity': 'SEVERITY', 'status': 'STATUS',
'duplicateCount': 'DUPL', 'customer': 'CUSTOMER', 'environment': 'ENVIRONMENT', 'service': 'SERVICE',
'resource': 'RESOURCE', 'group': 'GROUP', 'event': 'EVENT', 'value': 'VALUE', 'text': 'TEXT'}
click.echo(tabulate([a.tabular('summary', timezone)
for a in alerts], headers=headers, tablefmt=obj['output']))
elif display in ['compact', 'details']:
for alert in reversed(alerts):
color = COLOR_MAP.get(alert.severity, {'fg': 'white'})
click.secho('{}|{}|{}|{:5d}|{}|{:<5s}|{:<10s}|{:<18s}|{:12s}|{:16s}|{:12s}'.format(
alert.id[0:8],
DateTime.localtime(alert.last_receive_time, timezone),
alert.severity,
alert.duplicate_count,
alert.customer or '-',
alert.environment,
','.join(alert.service),
alert.resource,
alert.group,
alert.event,
alert.value or 'n/a'), fg=color['fg'])
click.secho(' |{}'.format(alert.text), fg=color['fg'])
if display == 'details':
click.secho(' severity | {} -> {}'.format(alert.previous_severity,
alert.severity), fg=color['fg'])
click.secho(' trend | {}'.format(alert.trend_indication), fg=color['fg'])
click.secho(' status | {}'.format(alert.status), fg=color['fg'])
click.secho(' resource | {}'.format(alert.resource), fg=color['fg'])
click.secho(' group | {}'.format(alert.group), fg=color['fg'])
click.secho(' event | {}'.format(alert.event), fg=color['fg'])
click.secho(' value | {}'.format(alert.value), fg=color['fg'])
click.secho(' tags | {}'.format(' '.join(alert.tags)), fg=color['fg'])
for key, value in alert.attributes.items():
click.secho(' {} | {}'.format(key.ljust(10), value), fg=color['fg'])
latency = alert.receive_time - alert.create_time
click.secho(' time created | {}'.format(
DateTime.localtime(alert.create_time, timezone)), fg=color['fg'])
click.secho(' time received | {}'.format(
DateTime.localtime(alert.receive_time, timezone)), fg=color['fg'])
click.secho(' last received | {}'.format(
DateTime.localtime(alert.last_receive_time, timezone)), fg=color['fg'])
click.secho(' latency | {}ms'.format((latency.microseconds / 1000)), fg=color['fg'])
click.secho(' timeout | {}s'.format(alert.timeout), fg=color['fg'])
click.secho(' alert id | {}'.format(alert.id), fg=color['fg'])
click.secho(' last recv id | {}'.format(alert.last_receive_id), fg=color['fg'])
click.secho(' customer | {}'.format(alert.customer), fg=color['fg'])
click.secho(' environment | {}'.format(alert.environment), fg=color['fg'])
click.secho(' service | {}'.format(','.join(alert.service)), fg=color['fg'])
click.secho(' resource | {}'.format(alert.resource), fg=color['fg'])
click.secho(' type | {}'.format(alert.event_type), fg=color['fg'])
click.secho(' repeat | {}'.format(alert.repeat), fg=color['fg'])
click.secho(' origin | {}'.format(alert.origin), fg=color['fg'])
click.secho(' correlate | {}'.format(','.join(alert.correlate)), fg=color['fg'])
return auto_refresh, last_time
def get_alerts(self, query=None, page=1, page_size=None):
r = self.http.get('/alerts', query, page=page, page_size=page_size)
return [Alert.parse(a) for a in r['alerts']]
def cli(obj, ids, filters, display, from_date=None):
"""Query for alerts based on search filter criteria."""
client = obj['client']
timezone = obj['timezone']
if ids:
query = [('id', x) for x in ids]
else:
query = build_query(filters)
if from_date:
query.append(('from-date', from_date))
r = client.http.get('/alerts', query)
if obj['output'] == 'json':
click.echo(
json.dumps(r['alerts'],
sort_keys=True,
indent=4,
ensure_ascii=False))
elif obj['output'] in ['json_lines', 'jsonl', 'ndjson']:
for alert in r['alerts']:
click.echo(json.dumps(alert, ensure_ascii=False))
else:
alerts = [Alert.parse(a) for a in r['alerts']]
last_time = r['lastTime']
auto_refresh = r['autoRefresh']
if display == 'tabular':
headers = {
'id': 'ID',
'lastReceiveTime': 'LAST RECEIVED',
'severity': 'SEVERITY',
'status': 'STATUS',
'duplicateCount': 'DUPL',
'customer': 'CUSTOMER',
'environment': 'ENVIRONMENT',
'service': 'SERVICE',
'resource': 'RESOURCE',
'group': 'GROUP',
'event': 'EVENT',
'value': 'VALUE',
'text': 'TEXT'
}
click.echo(
tabulate([a.tabular('summary', timezone) for a in alerts],
headers=headers,
tablefmt=obj['output']))
elif display in ['compact', 'details']:
for alert in reversed(alerts):
color = COLOR_MAP.get(alert.severity, {'fg': 'white'})
click.secho(
'{0}|{1}|{2}|{3:5d}|{4}|{5:<5s}|{6:<10s}|{7:<18s}|{8:12s}|{9:16s}|{10:12s}'
.format(
alert.id[0:8],
DateTime.localtime(alert.last_receive_time, timezone),
alert.severity, alert.duplicate_count, alert.customer
or "-", alert.environment, ','.join(alert.service),
alert.resource, alert.group, alert.event, alert.value
or "n/a"),
fg=color['fg'])
click.secho(' |{}'.format(alert.text), fg=color['fg'])
if display == 'details':
click.secho(' severity | {} -> {}'.format(
alert.previous_severity, alert.severity),
fg=color['fg'])
click.secho(' trend | {}'.format(
alert.trend_indication),
fg=color['fg'])
click.secho(' status | {}'.format(alert.status),
fg=color['fg'])
click.secho(' resource | {}'.format(alert.resource),
fg=color['fg'])
click.secho(' group | {}'.format(alert.group),
fg=color['fg'])
click.secho(' event | {}'.format(alert.event),
fg=color['fg'])
click.secho(' value | {}'.format(alert.value),
fg=color['fg'])
click.secho(' tags | {}'.format(' '.join(
alert.tags)),
fg=color['fg'])
for key, value in alert.attributes.items():
click.secho(' {} | {}'.format(key.ljust(10), value),
fg=color['fg'])
latency = alert.receive_time - alert.create_time
click.secho(' time created | {}'.format(
DateTime.localtime(alert.create_time, timezone)),
fg=color['fg'])
click.secho(' time received | {}'.format(
DateTime.localtime(alert.receive_time, timezone)),
fg=color['fg'])
click.secho(' last received | {}'.format(
DateTime.localtime(alert.last_receive_time, timezone)),
fg=color['fg'])
click.secho(' latency | {}ms'.format(
(latency.microseconds / 1000)),
fg=color['fg'])
click.secho(' timeout | {}s'.format(
alert.timeout),
fg=color['fg'])
click.secho(' alert id | {}'.format(
alert.id),
fg=color['fg'])
click.secho(' last recv id | {}'.format(
alert.last_receive_id),
fg=color['fg'])
click.secho(' customer | {}'.format(
alert.customer),
fg=color['fg'])
click.secho(' environment | {}'.format(
alert.environment),
fg=color['fg'])
click.secho(' service | {}'.format(
','.join(alert.service)),
fg=color['fg'])
click.secho(' resource | {}'.format(
alert.resource),
fg=color['fg'])
click.secho(' type | {}'.format(
alert.event_type),
fg=color['fg'])
click.secho(' repeat | {}'.format(
alert.repeat),
fg=color['fg'])
click.secho(' origin | {}'.format(
alert.origin),
fg=color['fg'])
click.secho(' correlate | {}'.format(
','.join(alert.correlate)),
fg=color['fg'])
return auto_refresh, last_time