def housekeeping(expired_threshold: int = 2,
info_threshold: int = 12) -> None:
expired, unshelved = db.housekeeping(expired_threshold, info_threshold)
for (id, event, last_receive_id) in expired:
history = History(id=last_receive_id,
event=event,
status='expired',
text='expired after timeout',
change_type='status',
update_time=datetime.utcnow())
db.set_status(id,
'expired',
timeout=current_app.config['ALERT_TIMEOUT'],
history=history)
for (id, event, last_receive_id) in unshelved:
history = History(id=last_receive_id,
event=event,
status='open',
text='unshelved after timeout',
change_type='status',
update_time=datetime.utcnow())
db.set_status(id,
'open',
timeout=current_app.config['ALERT_TIMEOUT'],
history=history)
def housekeeping(expired_threshold: int=2, info_threshold: int=12) -> None:
now = datetime.utcnow()
expired, unshelved = db.housekeeping(expired_threshold, info_threshold)
for (id, event, last_receive_id) in expired:
history = History(
id=last_receive_id,
event=event,
status='expired',
text='auto-expired after timeout',
change_type='status',
update_time=now,
user=g.login
)
db.set_status(id, 'expired', timeout=current_app.config['ALERT_TIMEOUT'], update_time=now, history=history)
for (id, event, last_receive_id) in unshelved:
# as per ISA 18.2 recommendation 11.7.3 auto-unshelved alarms transition to open, not previous status
history = History(
id=last_receive_id,
event=event,
status='open',
text='auto-unshelved after timeout',
change_type='status',
update_time=now,
user=g.login
)
db.set_status(id, 'open', timeout=current_app.config['ALERT_TIMEOUT'], update_time=now, history=history)
def housekeeping(expired_threshold: int = 2, info_threshold: int = 12) -> None:
now = datetime.utcnow()
expired, unshelved = db.housekeeping(expired_threshold, info_threshold)
for (id, event, last_receive_id) in expired:
history = History(
id=last_receive_id,
event=event,
status='expired',
text='auto-expired after timeout',
change_type='status',
update_time=now,
user=g.login
)
db.set_status(id, 'expired', timeout=current_app.config['ALERT_TIMEOUT'], update_time=now, history=history)
for (id, event, last_receive_id) in unshelved:
# as per ISA 18.2 recommendation 11.7.3 auto-unshelved alarms transition to open, not previous status
history = History(
id=last_receive_id,
event=event,
status='open',
text='auto-unshelved after timeout',
change_type='status',
update_time=now,
user=g.login
)
db.set_status(id, 'open', timeout=current_app.config['ALERT_TIMEOUT'], update_time=now, history=history)
def housekeeping(expired_threshold=2, info_threshold=12):
for (id, event, last_receive_id) in db.housekeeping(expired_threshold, info_threshold):
history = History(
id=last_receive_id,
event=event,
status="expired",
text="alert timeout status change",
change_type="status",
update_time=datetime.utcnow()
)
db.set_status(id, "expired", history)
def housekeeping(expired_threshold=2, info_threshold=12):
for (id, event, last_receive_id) in db.housekeeping(expired_threshold, info_threshold):
history = History(
id=last_receive_id,
event=event,
status="expired",
text="alert timeout status change",
change_type="status",
update_time=datetime.utcnow()
)
db.set_status(id, "expired", history)
def set_status(id):
# FIXME - should only allow role=user to set status for alerts for that customer
# Above comment is from original code, can ignore it for now
status_started = status_timer.start_timer()
tenant = getTenantFromHeader(request)
if len(tenant) == 0:
return jsonify(status="error", message="bad request"), 400
data = request.json
tenant = generateDBName(tenant)
if data and 'status' in data:
try:
alert = db.set_status(tenant, id=id, status=data['status'], text=data.get('text', ''))
except Exception as e:
return jsonify(status="error", message=str(e)), 500
else:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="must supply 'status' as parameter"), 400
if alert:
status_timer.stop_timer(status_started)
return jsonify(status="ok")
else:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="not found"), 404
def set_status(id):
# FIXME - should only allow role=user to set status for alerts for that customer
status_started = status_timer.start_timer()
data = request.json
if data and 'status' in data:
try:
alert = db.set_status(id=id,
status=data['status'],
text=data.get('text', ''))
except Exception as e:
return jsonify(status="error", message=str(e)), 500
else:
status_timer.stop_timer(status_started)
return jsonify(status="error",
message="must supply 'status' as parameter"), 400
if alert:
status_timer.stop_timer(status_started)
return jsonify(status="ok")
else:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="not found"), 404
def pagerduty():
hook_started = webhook_timer.start_timer()
data = request.json
if data and 'messages' in data:
for message in data['messages']:
try:
id, status, text = parse_pagerduty(message)
except IndexError as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 400
try:
alert = db.set_status(id=id, status=status, text=text)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
else:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message="no messages in PagerDuty data payload"), 400
webhook_timer.stop_timer(hook_started)
if alert:
return jsonify(status="ok"), 200
else:
return jsonify(status="error", message="update PagerDuty incident status failed"), 500
def pagerduty():
hook_started = webhook_timer.start_timer()
data = request.json
if data and 'messages' in data:
for message in data['messages']:
try:
id, status, text = parse_pagerduty(message)
except IndexError as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 400
try:
alert = db.set_status(id=id, status=status, text=text)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
else:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error",
message="no messages in PagerDuty data payload"), 400
webhook_timer.stop_timer(hook_started)
if alert:
return jsonify(status="ok"), 200
else:
return jsonify(status="error",
message="update PagerDuty incident status failed"), 500
def telegram():
data = request.json
if 'callback_query' in data:
command, alert = data['callback_query']['data'].split(' ', 1)
if command == '/ack':
db.set_status(alert, 'ack', 'status change via Telegram')
elif command == '/close':
db.set_status(alert, 'closed', 'status change via Telegram')
elif command == '/blackout':
environment, resource, event = alert.split('|', 2)
db.create_blackout(environment,resource=resource, event=event)
return jsonify(status="ok")
else:
return jsonify(status="error", message="no callback_query in Telegram message"), 400
def telegram():
data = request.json
if 'callback_query' in data:
command, alert = data['callback_query']['data'].split(' ', 1)
if command == '/ack':
db.set_status(alert, 'ack', 'status change via Telegram')
elif command == '/close':
db.set_status(alert, 'closed', 'status change via Telegram')
elif command == '/blackout':
environment, resource, event = alert.split('|', 2)
db.create_blackout(environment,resource=resource, event=event)
return jsonify(status="ok")
else:
return jsonify(status="error", message="no callback_query in Telegram message"), 400
def set_status(self, status, text=''):
history = History(id=self.id,
event=self.event,
status=status,
text=text,
change_type="status",
update_time=datetime.utcnow())
return db.set_status(self.id, status, history)
def set_status(self, status, text='', timeout=None):
timeout = timeout or current_app.config['ALERT_TIMEOUT']
history = History(id=self.id,
event=self.event,
status=status,
text=text,
change_type='status',
update_time=datetime.utcnow())
return db.set_status(self.id, status, timeout, history)
def set_status(self, status, text=''):
history = History(
id=self.id,
event=self.event,
status=status,
text=text,
change_type="status",
update_time=datetime.utcnow()
)
return db.set_status(self.id, status, history)
def housekeeping(expired_threshold=2, info_threshold=12):
for (id, event, status,
last_receive_id) in db.housekeeping(expired_threshold,
info_threshold):
if status == 'open':
text = 'unshelved after timeout'
elif status == 'expired':
text = 'expired after timeout'
else:
text = 'alert timeout status change'
history = History(id=last_receive_id,
event=event,
status=status,
text=text,
change_type="status",
update_time=datetime.utcnow())
db.set_status(id,
status,
timeout=current_app.config['ALERT_TIMEOUT'],
history=history)
def set_status(self, status: str, text: str='', timeout: int=None) -> 'Alert':
timeout = timeout or current_app.config['ALERT_TIMEOUT']
history = History(
id=self.id,
event=self.event,
severity=self.severity,
status=status,
value=self.value,
text=text,
change_type='status',
update_time=datetime.utcnow()
)
return db.set_status(self.id, status, timeout, history)
def set_status(id):
data = request.json
if data and 'status' in data:
alert = db.set_status(id=id, status=data['status'], text=data.get('text', ''))
else:
return jsonify(status="error", message="no data")
if alert:
notify.send(alert)
return jsonify(status="ok")
else:
return jsonify(status="error", message="failed to set alert status")
def telegram():
data = request.json
if 'callback_query' in data:
author = data['callback_query']['from']
command, alert = data['callback_query']['data'].split(' ', 1)
user = "******".format(author.get('first_name'),
author.get('last_name'))
action = command.lstrip('/')
if action in ['open', 'ack', 'close']:
db.set_status(alert, action, 'status change via Telegram')
elif action in ['watch', 'unwatch']:
db.untag_alert(alert, [
"{}:{}".format(action, user),
])
elif action == 'blackout':
environment, resource, event = alert.split('|', 2)
db.create_blackout(environment, resource=resource, event=event)
send_message_reply(alert, action, user, data)
return jsonify(status="ok")
else:
return jsonify(status="error",
message="no callback_query in Telegram message"), 400
def set_status(self, status: str, text: str='', timeout: int=None) -> 'Alert':
now = datetime.utcnow()
timeout = timeout or current_app.config['ALERT_TIMEOUT']
history = History(
id=self.id,
event=self.event,
severity=self.severity,
status=status,
value=self.value,
text=text,
change_type='status',
update_time=now,
user=g.login
)
return db.set_status(self.id, status, timeout, update_time=now, history=history)
def set_status(id):
status_started = status_timer.start_timer()
customer = g.get('customer', None)
try:
alert = db.get_alert(id=id, customer=customer)
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
status_timer.stop_timer(status_started)
return jsonify(status="error",
message="not found",
total=0,
alert=None), 404
status = request.json.get('status', None)
text = request.json.get('text', '')
if not status:
status_timer.stop_timer(status_started)
return jsonify(status="error",
message="must supply 'status' as parameter"), 400
try:
process_status(alert, status, text)
except RejectException as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 403
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
try:
alert = db.set_status(id=id, status=status, text=text)
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
if alert:
status_timer.stop_timer(status_started)
return jsonify(status="ok")
else:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="not found"), 404
def set_status(self, status: str, text: str = '', timeout: int = None) -> 'Alert':
now = datetime.utcnow()
timeout = timeout or current_app.config['ALERT_TIMEOUT']
history = History(
id=self.id,
event=self.event,
severity=self.severity,
status=status,
value=self.value,
text=text,
change_type=ChangeType.status,
update_time=now,
user=g.login,
timeout=self.timeout
)
return db.set_status(self.id, status, timeout, update_time=now, history=history)
def pagerduty():
hook_started = webhook_timer.start_timer()
data = request.json
if data and 'messages' in data:
for message in data['messages']:
try:
id, status, text = parse_pagerduty(message)
except IndexError, e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 400
try:
alert = db.set_status(id=id, status=status, text=text)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
def set_status(id):
status_started = status_timer.start_timer()
customer = g.get('customer', None)
try:
alert = db.get_alert(id=id, customer=customer)
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="not found", total=0, alert=None), 404
status = request.json.get('status', None)
text = request.json.get('text', '')
if not status:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="must supply 'status' as parameter"), 400
try:
process_status(alert, status, text)
except RejectException as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 403
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
try:
alert = db.set_status(id=id, status=status, text=text)
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
if alert:
status_timer.stop_timer(status_started)
return jsonify(status="ok")
else:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="not found"), 404
def set_status(id):
status_started = status_timer.start_timer()
data = request.json
if data and 'status' in data:
try:
alert = db.set_status(id=id, status=data['status'], text=data.get('text', ''))
except Exception as e:
return jsonify(status="error", message=str(e)), 500
else:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="must supply 'status' as parameter"), 400
if alert:
status_timer.stop_timer(status_started)
return jsonify(status="ok")
else:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="not found"), 404
def stackdriver():
hook_started = webhook_timer.start_timer()
try:
state, incomingAlert = parse_stackdriver(request.data)
except ValueError as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 400
if g.get('customer', None):
incomingAlert.customer = g.get('customer')
if state == 'acknowledged':
try:
alert = db.set_status(id=incomingAlert.id,
status='ack',
text='acknowledged via Stackdriver')
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
else:
try:
alert = process_alert(incomingAlert)
except RejectException as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 403
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
webhook_timer.stop_timer(hook_started)
if alert:
body = alert.get_body()
body['href'] = absolute_url('/alert/' + alert.id)
return jsonify(status="ok", id=alert.id, alert=body), 201, {
'Location': body['href']
}
else:
return jsonify(status="error",
message="notification from stackdriver failed"), 500
def pagerduty():
hook_started = webhook_timer.start_timer()
data = request.json
updated = False
if data and 'messages' in data:
for message in data['messages']:
try:
incident_key, status, text = parse_pagerduty(message)
except ValueError as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 400
customer = g.get('customer', None)
try:
alert = db.get_alert(id=incident_key, customer=customer)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
webhook_timer.stop_timer(hook_started)
return jsonify(stats="error", message="not found"), 404
try:
updated = db.set_status(id=alert.id, status=status, text=text)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
else:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error",
message="no messages in PagerDuty data payload"), 400
webhook_timer.stop_timer(hook_started)
if updated:
return jsonify(status="ok"), 200
else:
return jsonify(status="error",
message="update PagerDuty incident status failed"), 500
def pagerduty():
hook_started = webhook_timer.start_timer()
data = request.json
updated = False
if data and 'messages' in data:
for message in data['messages']:
try:
incident_key, status, text = parse_pagerduty(message)
except ValueError as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 400
customer = g.get('customer', None)
try:
alert = db.get_alert(id=incident_key, customer=customer)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
webhook_timer.stop_timer(hook_started)
return jsonify(stats="error", message="not found"), 404
try:
updated = db.set_status(id=alert.id, status=status, text=text)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
else:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message="no messages in PagerDuty data payload"), 400
webhook_timer.stop_timer(hook_started)
if updated:
return jsonify(status="ok"), 200
else:
return jsonify(status="error", message="update PagerDuty incident status failed"), 500
def stackdriver():
hook_started = webhook_timer.start_timer()
try:
state, incomingAlert = parse_stackdriver(request.data)
except ValueError as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 400
if g.get('customer', None):
incomingAlert.customer = g.get('customer')
if state == 'acknowledged':
try:
alert = db.set_status(id=incomingAlert.id, status='ack', text='acknowledged via Stackdriver')
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
else:
try:
alert = process_alert(incomingAlert)
except RejectException as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 403
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
webhook_timer.stop_timer(hook_started)
if alert:
body = alert.get_body()
body['href'] = absolute_url('/alert/' + alert.id)
return jsonify(status="ok", id=alert.id, alert=body), 201, {'Location': body['href']}
else:
return jsonify(status="error", message="notification from stackdriver failed"), 500
def slack():
hook_started = webhook_timer.start_timer()
try:
alert, user, action = parse_slack(request.form)
except ValueError as e:
webhook_timer.stop_timer(hook_started)
return jsonify(stats="error", message="not found"), 404
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
if action in ['open', 'ack', 'close']:
try:
alert = db.set_status(
alert.id, action,
u"status change via Slack by {}".format(user))
except RejectException as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 403
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
elif action in ['watch', 'unwatch']:
db.untag_alert(alert.id, [
"{}:{}".format(action, user),
])
else:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=u'Unsuported action'), 403
response = build_slack_response(alert, action, user, request.form)
webhook_timer.stop_timer(hook_started)
return jsonify(**response), 201
