def bulk_set_status():
status = request.json.get('status', None)
text = request.json.get('text', 'bulk status update')
timeout = request.json.get('timeout', None)
if not status:
raise ApiError("must supply 'status' as json data", 400)
query = qb.from_params(request.args)
alerts = Alert.find_all(query)
if not alerts:
raise ApiError('not found', 404)
updated = []
errors = []
for alert in alerts:
try:
alert, status, text = process_status(alert, status, text)
except RejectException as e:
errors.append(str(e))
continue
except Exception as e:
errors.append(str(e))
continue
if alert.set_status(status, text, timeout):
updated.append(alert.id)
if errors:
raise ApiError('failed to bulk set alert status', 500, errors=errors)
else:
return jsonify(status='ok', updated=updated, count=len(updated))
def incoming(self, query_string, payload):
if payload and payload['state'] == 'alerting':
return [parse_grafana(payload, match, query_string) for match in payload.get('evalMatches', [])]
elif payload and payload['state'] == 'ok' and payload.get('ruleId'):
try:
query = qb.from_dict({'attributes.ruleId': str(payload['ruleId'])})
existingAlerts = Alert.find_all(query)
except Exception as e:
raise ApiError(str(e), 500)
alerts = []
for updateAlert in existingAlerts:
updateAlert.severity = 'normal'
updateAlert.status = 'closed'
try:
alert = process_alert(updateAlert)
except RejectException as e:
raise ApiError(str(e), 403)
except Exception as e:
raise ApiError(str(e), 500)
alerts.append(alert)
return alerts
else:
raise ApiError('no alerts in Grafana notification payload', 400)
def blackout_change(self, blackout, action):
LOG.debug('Blackout {} state changed: {}'.format(blackout.id, action))
alerts = Alert.find_all()
for alert in alerts:
if action == 'create' and alert.is_blackout():
LOG.debug("{} alert should be muted".format(alert.id))
attributes = alert.attributes
attributes['blackout'] = blackout.id
alert.update_attributes(attributes)
alert.set_status('blackout', text='muted by blackout plugin')
elif action == 'update':
if alert.is_blackout():
if alert.status != 'blackout':
attributes = alert.attributes
attributes['blackout'] = blackout.id
alert.update_attributes(attributes)
alert.set_status('blackout',
text='muted by blackout plugin')
elif 'blackout' in alert.attributes:
alert.set_status('open',
text='reopened by blackout plugin')
elif action == 'delete':
if 'blackout' in alert.attributes and alert.attributes[
'blackout'] == blackout.id:
alert.set_status('open',
text='reopened by blackout plugin')
def bulk_set_status():
status = request.json.get('status', None)
text = request.json.get('text', 'bulk status update')
timeout = request.json.get('timeout', None)
if not status:
raise ApiError("must supply 'status' as json data", 400)
query = qb.from_params(request.args)
alerts = Alert.find_all(query)
if not alerts:
raise ApiError('not found', 404)
updated = []
errors = []
for alert in alerts:
try:
alert, status, text = process_status(alert, status, text)
except RejectException as e:
errors.append(str(e))
continue
except Exception as e:
errors.append(str(e))
continue
if alert.set_status(status, text, timeout):
updated.append(alert.id)
if errors:
raise ApiError('failed to bulk set alert status', 500, errors=errors)
else:
return jsonify(status='ok', updated=updated, count=len(updated))
def incoming(self, path, query_string, payload):
if payload and payload['state'] == 'alerting':
return [parse_grafana(query_string, payload, match) for match in payload.get('evalMatches', [])]
elif payload and payload['state'] == 'ok' and payload.get('ruleId'):
try:
query = qb.alerts.from_params(MultiDict([('attributes.ruleId', str(payload['ruleId']))]))
existingAlerts = Alert.find_all(query)
except Exception as e:
raise ApiError(str(e), 500)
alerts = []
for updateAlert in existingAlerts:
updateAlert.severity = alarm_model.DEFAULT_NORMAL_SEVERITY
updateAlert.status = 'closed'
try:
alert = process_alert(updateAlert)
except RejectException as e:
raise ApiError(str(e), 403)
except Exception as e:
raise ApiError(str(e), 500)
alerts.append(alert)
return alerts
else:
raise ApiError('no alerts in Grafana notification payload', 400)
def search_alerts():
query_time = datetime.utcnow()
query = qb.from_params(request.args,
customers=g.customers,
query_time=query_time)
show_raw_data = request.args.get(
'show-raw-data',
default=False,
type=lambda x: x.lower() in ['true', 't', '1', 'yes', 'y', 'on'])
show_history = request.args.get(
'show-history',
default=False,
type=lambda x: x.lower() in ['true', 't', '1', 'yes', 'y', 'on'])
severity_count = Alert.get_counts_by_severity(query)
status_count = Alert.get_counts_by_status(query)
total = sum(severity_count.values())
paging = Page.from_params(request.args, total)
alerts = Alert.find_all(query, paging.page, paging.page_size)
for alert in alerts:
if not show_raw_data:
alert.raw_data = None
if not show_history:
alert.history = []
if alerts:
return jsonify(
status='ok',
page=paging.page,
pageSize=paging.page_size,
pages=paging.pages,
more=paging.has_more,
alerts=[alert.serialize for alert in alerts],
total=total,
statusCounts=status_count,
severityCounts=severity_count,
lastTime=max([alert.last_receive_time for alert in alerts]),
autoRefresh=Switch.find_by_name('auto-refresh-allow').is_on)
else:
return jsonify(
status='ok',
message='not found',
page=paging.page,
pageSize=paging.page_size,
pages=0,
more=False,
alerts=[],
total=0,
severityCounts=severity_count,
statusCounts=status_count,
lastTime=query_time,
autoRefresh=Switch.find_by_name('auto-refresh-allow').is_on)
def grafana():
alerts = []
data = request.json
if data and data['state'] == 'alerting':
for match in data.get('evalMatches', []):
try:
incomingAlert = parse_grafana(data, match, request.args)
except ValueError as e:
return jsonify(status='error', message=str(e)), 400
incomingAlert.customer = assign_customer(wanted=incomingAlert.customer)
add_remote_ip(request, incomingAlert)
try:
alert = process_alert(incomingAlert)
except RejectException as e:
return jsonify(status='error', message=str(e)), 403
except Exception as e:
return jsonify(status='error', message=str(e)), 500
alerts.append(alert)
elif data and data['state'] == 'ok' and data.get('ruleId', None):
try:
query = qb.from_dict({'attributes.ruleId': str(data['ruleId'])})
existingAlerts = Alert.find_all(query)
except Exception as e:
raise ApiError(str(e), 500)
for updateAlert in existingAlerts:
updateAlert.severity = 'normal'
updateAlert.status = 'closed'
try:
alert = process_alert(updateAlert)
except RejectException as e:
raise ApiError(str(e), 403)
except Exception as e:
raise ApiError(str(e), 500)
alerts.append(alert)
else:
raise ApiError('no alerts in Grafana notification payload', 400)
for alert in alerts:
text = 'grafana alert received via webhook'
write_audit_trail.send(current_app._get_current_object(), event='webhook-received', message=text, user=g.user,
customers=g.customers, scopes=g.scopes, resource_id=alert.id, type='alert', request=request)
if len(alerts) == 1:
return jsonify(status='ok', id=alerts[0].id, alert=alerts[0].serialize), 201
else:
return jsonify(status='ok', ids=[alert.id for alert in alerts]), 201
def grafana():
alerts = []
data = request.json
if data and data['state'] == 'alerting':
for match in data.get('evalMatches', []):
try:
incomingAlert = parse_grafana(data, match)
except ValueError as e:
return jsonify(status="error", message=str(e)), 400
if g.get('customer', None):
incomingAlert.customer = g.get('customer')
add_remote_ip(request, incomingAlert)
try:
alert = process_alert(incomingAlert)
except RejectException as e:
return jsonify(status="error", message=str(e)), 403
except Exception as e:
return jsonify(status="error", message=str(e)), 500
alerts.append(alert)
elif data and data['state'] == 'ok' and data.get('ruleId', None):
try:
query = qb.from_dict({'attributes.ruleId': str(data['ruleId'])})
existingAlerts = Alert.find_all(query)
except Exception as e:
raise ApiError(str(e), 500)
for updateAlert in existingAlerts:
updateAlert.severity = 'normal'
updateAlert.status = 'closed'
try:
alert = process_alert(updateAlert)
except RejectException as e:
raise ApiError(str(e), 403)
except Exception as e:
raise ApiError(str(e), 500)
alerts.append(alert)
else:
raise ApiError("no alerts in Grafana notification payload", 400)
if len(alerts) == 1:
return jsonify(status="ok", id=alerts[0].id,
alert=alerts[0].serialize), 201
else:
return jsonify(status="ok", ids=[alert.id for alert in alerts]), 201
def grafana():
alerts = []
data = request.json
if data and data['state'] == 'alerting':
for match in data.get('evalMatches', []):
try:
incomingAlert = parse_grafana(data, match)
except ValueError as e:
return jsonify(status="error", message=str(e)), 400
if g.get('customer', None):
incomingAlert.customer = g.get('customer')
add_remote_ip(request, incomingAlert)
try:
alert = process_alert(incomingAlert)
except RejectException as e:
return jsonify(status="error", message=str(e)), 403
except Exception as e:
return jsonify(status="error", message=str(e)), 500
alerts.append(alert)
elif data and data['state'] == 'ok' and data.get('ruleId', None):
try:
query = qb.from_dict({'attributes.ruleId': str(data['ruleId'])})
existingAlerts = Alert.find_all(query)
except Exception as e:
raise ApiError(str(e), 500)
for updateAlert in existingAlerts:
updateAlert.severity = 'normal'
updateAlert.status = 'closed'
try:
alert = process_alert(updateAlert)
except RejectException as e:
raise ApiError(str(e), 403)
except Exception as e:
raise ApiError(str(e), 500)
alerts.append(alert)
else:
raise ApiError("no alerts in Grafana notification payload", 400)
if len(alerts) == 1:
return jsonify(status="ok", id=alerts[0].id, alert=alerts[0].serialize), 201
else:
return jsonify(status="ok", ids=[alert.id for alert in alerts]), 201
def bulk_action_alert():
from alerta.tasks import action_alerts
action = request.json.get('action', None)
text = request.json.get('text', 'bulk status update')
timeout = request.json.get('timeout', None)
if not action:
raise ApiError("must supply 'action' as json data", 400)
query = qb.from_params(request.args)
alerts = [alert.id for alert in Alert.find_all(query)]
if not alerts:
raise ApiError('not found', 404)
task = action_alerts.delay(alerts, action, text, timeout)
return jsonify(status='ok', message='{} alerts queued for action'.format(len(alerts))), 202, {'Location': absolute_url('/_bulk/task/' + task.id)}
def search_alerts():
query_time = datetime.utcnow()
query = qb.from_params(request.args,
customers=g.customers,
query_time=query_time)
severity_count = Alert.get_counts_by_severity(query)
status_count = Alert.get_counts_by_status(query)
total = sum(severity_count.values())
paging = Page.from_params(request.args, total)
alerts = Alert.find_all(query, paging.page, paging.page_size)
if alerts:
return jsonify(
status='ok',
page=paging.page,
pageSize=paging.page_size,
pages=paging.pages,
more=paging.has_more,
alerts=[alert.serialize for alert in alerts],
total=total,
statusCounts=status_count,
severityCounts=severity_count,
lastTime=max([alert.last_receive_time for alert in alerts]),
autoRefresh=Switch.find_by_name('auto-refresh-allow').is_on)
else:
return jsonify(
status='ok',
message='not found',
page=paging.page,
pageSize=paging.page_size,
pages=0,
more=False,
alerts=[],
total=0,
severityCounts=severity_count,
statusCounts=status_count,
lastTime=query_time,
autoRefresh=Switch.find_by_name('auto-refresh-allow').is_on)
def search_alerts():
query_time = datetime.utcnow()
query = qb.from_params(request.args, query_time)
severity_count = Alert.get_counts_by_severity(query)
status_count = Alert.get_counts_by_status(query)
total = sum(severity_count.values())
paging = Page.from_params(request.args, total)
alerts = Alert.find_all(query, paging.page, paging.page_size)
if alerts:
return jsonify(
status="ok",
page=paging.page,
pageSize=paging.page_size,
pages=paging.pages,
more=paging.has_more,
alerts=[alert.serialize for alert in alerts],
total=total,
statusCounts=status_count,
severityCounts=severity_count,
lastTime=max([alert.last_receive_time for alert in alerts]),
autoRefresh=Switch.find_by_name('auto-refresh-allow').is_on
)
else:
return jsonify(
status="ok",
message="not found",
page=paging.page,
pageSize=paging.page_size,
pages=0,
more=False,
alerts=[],
total=0,
severityCounts=severity_count,
statusCounts=status_count,
lastTime=query_time,
autoRefresh=Switch.find_by_name('auto-refresh-allow').is_on
)
def post_receive(self, alert):
# еластик алерты не тухнут по событию ОК, а только по таймауту. поэтому всегда считаются дублирующими, это надо чтобы уведомления всетаки приходили
if alert.group == 'elastalert' and alert.history[0].status == 'expired':
alert.repeat = False
# этот кусок кода для того чтобы одноименные события, которые из WARNING поднялись выше по severity таже не считались повторами и уведомление приходило
if alert.severity in TELEGRAM_SOUND_NOTIFICATION_SEVERITY and alert.history[
0].severity == 'warning':
alert.repeat = False
if alert.severity == 'critical' and alert.group != 'elastalert':
alert.repeat = False
if alert.repeat:
return
alert.create_time = alert.create_time + timedelta(hours=3)
alert.update_time = alert.update_time + timedelta(hours=3)
alert.service = ', '.join(alert.service)
try:
raw = json.loads(alert.raw_data)
if 'ruleUrl' in raw.keys():
alert.attributes['ruleUrl'] = raw['ruleUrl']
if 'incident_url' in raw.keys():
alert.attributes['incident_url'] = raw['incident_url']
except:
LOG.debug('DEBUG:', 'No raw fileld for parse:')
try:
text = self.template.render(alert.__dict__)
except UndefinedError:
text = "Something bad has happened but also we " \
"can't handle your telegram template message."
LOG.debug('Telegram: message=%s', text)
if TELEGRAM_WEBHOOK_URL and alert.status == 'open':
keyboard = {
'inline_keyboard': [[{
'text':
'📄 Подробнее',
'url':
'https://alerta.docdoc.pro/alert/' + alert.id
}, {
'text': '⛑ Подтвердить',
'callback_data': '/ack ' + alert.id
}, {
'text': '🔇 Заглушить',
'callback_data': '/blackout ' + alert.id
}]]
}
if alert.group == 'elastalert':
keyboard['inline_keyboard'][0].append({
'text':
'❌ Закрыть',
'callback_data':
'/close ' + alert.id
})
else:
keyboard = None
LOG.debug('Telegram inline_keyboard: %s', keyboard)
if TELEGRAM_SOUND_NOTIFICATION_SEVERITY:
disable_notification = True
if alert.severity in TELEGRAM_SOUND_NOTIFICATION_SEVERITY:
disable_notification = False
for i in alert.history:
if i.severity in TELEGRAM_SOUND_NOTIFICATION_SEVERITY:
disable_notification = False
break
else:
disable_notification = False
# inhibit
with open("/app/inhibit.yaml", 'r') as stream:
try:
inhibit_rules = yaml.safe_load(stream)
except yaml.YAMLError as exc:
print(exc)
Query = namedtuple('Query', ['where', 'vars', 'sort', 'group'])
Query.__new__.__defaults__ = ('1=1', {}, 'last_receive_time', 'status'
) # type: ignore
for name in list(inhibit_rules):
rule = inhibit_rules[name]
LOG.debug('inhibit_debug: %s', name)
if rule['dependent']:
try:
query = ['1=1']
qvars = dict()
query.append("AND status='open'")
query.append("AND " + rule['link_field'] + "=\'" +
str(alert.__dict__[rule['link_field']]) +
"\'")
query.append("AND id!=\'" + str(alert.id) + "\'")
query.append("AND " + rule['find_field'] + " ~ \'" +
rule['find_regexp'] + "\'")
LOG.debug('inhibit_debug: %s', ' '.join(query))
if Alert.find_all(Query(' '.join(query), qvars, None,
None)):
disable_notification = True
except:
LOG.debug('inhibit_debug ERROR')
else:
try:
if re.search(rule['find_regexp'],
str(alert.__dict__[rule['find_field']])
) and re.search(
rule['main_regexp'],
str(alert.__dict__[rule['main_field']])):
disable_notification = True
except:
LOG.debug('inhibit_debug ERROR')
LOG.debug('Telegram: post_receive sendMessage disable_notification=%s',
str(disable_notification) + ' ' + alert.severity)
if not disable_notification:
for count_try in range(10):
try:
if count_try > 1:
sleep(10)
response = self.bot.sendMessage(
TELEGRAM_CHAT_ID,
text,
parse_mode='Markdown',
disable_notification=disable_notification,
disable_web_page_preview=True,
reply_markup=keyboard)
except telepot.exception.TelegramError as e:
keyboard = None
LOG.debug(
"Telegram: ERROR - %s - %s, description= %s, json=%s",
count_try, e.error_code, e.description, e.json)
except Exception as e:
LOG.debug("Telegram: ERROR %s - %s, text: %s, keybord: %s",
count_try, e, text, keyboard)
except:
LOG.debug(
"Telegram: ERROR UNDEFINED, text: %s, keybord: %s",
text, keyboard)
else:
break
else:
raise LOG.debug(
"Telegram: ERROR after 10 try, text: %s, keybord: %s",
text, keyboard)
LOG.debug('Telegram response: %s', response)