def test_pagerduty_webhook(self):
# trigger alert
response = self.app.post('/alert',
data=json.dumps(self.trigger_alert),
headers=self.headers)
self.assertEqual(response.status_code, 201)
data = json.loads(response.data.decode('utf-8'))
trigger_alert_id = data['id']
# resolve alert
response = self.app.post('/alert',
data=json.dumps(self.resolve_alert),
headers=self.headers)
self.assertEqual(response.status_code, 201)
data = json.loads(response.data.decode('utf-8'))
resolve_alert_id = data['id']
response = self.app.post('/webhooks/pagerduty',
data=self.pagerduty_alert %
(trigger_alert_id, resolve_alert_id),
headers=self.headers)
self.assertEqual(response.status_code, 200)
self.assertEqual(db.get_alert(trigger_alert_id).status, "ack")
self.assertEqual(db.get_alert(resolve_alert_id).status, "closed")
def untag_alert(id):
untag_started = untag_timer.start_timer()
customer = g.get('customer', None)
try:
alert = db.get_alert(id=id, customer=customer)
except Exception as e:
untag_timer.stop_timer(untag_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
untag_timer.stop_timer(untag_started)
return jsonify(status="error",
message="not found",
total=0,
alert=None), 404
data = request.json
if data and 'tags' in data:
try:
response = db.untag_alert(id, data['tags'])
except Exception as e:
return jsonify(status="error", message=str(e)), 500
else:
untag_timer.stop_timer(untag_started)
return jsonify(status="error",
message="must supply 'tags' as list parameter"), 400
untag_timer.stop_timer(untag_started)
if response:
return jsonify(status="ok")
else:
return jsonify(status="error", message="not found"), 404
def untag_alert(id):
untag_started = untag_timer.start_timer()
customer = g.get('customer', None)
try:
alert = db.get_alert(id=id, customer=customer)
except Exception as e:
untag_timer.stop_timer(untag_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
untag_timer.stop_timer(untag_started)
return jsonify(status="error", message="not found", total=0, alert=None), 404
data = request.json
if data and 'tags' in data:
try:
response = db.untag_alert(id, data['tags'])
except Exception as e:
return jsonify(status="error", message=str(e)), 500
else:
untag_timer.stop_timer(untag_started)
return jsonify(status="error", message="must supply 'tags' as list parameter"), 400
untag_timer.stop_timer(untag_started)
if response:
return jsonify(status="ok")
else:
return jsonify(status="error", message="not found"), 404
def update_attributes(id):
attrs_started = attrs_timer.start_timer()
customer = g.get('customer', None)
try:
alert = db.get_alert(id=id, customer=customer)
except Exception as e:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="error", message="not found", total=0, alert=None), 404
attributes = request.json.get('attributes', None)
if not attributes:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="error", message="must supply 'attributes' as parameter"), 400
try:
alert = db.update_attributes(id, attributes)
except Exception as e:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="error", message=str(e)), 500
if alert:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="ok")
else:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="error", message="not found"), 404
def update_attributes(id):
attrs_started = attrs_timer.start_timer()
customer = g.get('customer', None)
try:
alert = db.get_alert(id=id, customer=customer)
except Exception as e:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="error", message="not found", total=0, alert=None), 404
attributes = request.json.get('attributes', None)
if not attributes:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="error", message="must supply 'attributes' as parameter"), 400
try:
alert = db.update_attributes(id, attributes)
except Exception as e:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="error", message=str(e)), 500
if alert:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="ok")
else:
attrs_timer.stop_timer(attrs_started)
return jsonify(status="error", message="not found"), 404
def get_alert(id):
alert = db.get_alert(id=id)
if alert:
return jsonify(status="ok", total=1, alert=alert.get_body())
else:
return jsonify(status="ok", message="not found", total=0, alert=None)
def test_pagerduty_webhook(self):
# trigger alert
response = self.app.post('/alert', data=json.dumps(self.trigger_alert), headers=self.headers)
self.assertEqual(response.status_code, 201)
data = json.loads(response.data.decode('utf-8'))
trigger_alert_id = data['id']
# resolve alert
response = self.app.post('/alert', data=json.dumps(self.resolve_alert), headers=self.headers)
self.assertEqual(response.status_code, 201)
data = json.loads(response.data.decode('utf-8'))
resolve_alert_id = data['id']
response = self.app.post('/webhooks/pagerduty', data=self.pagerduty_alert % (trigger_alert_id, resolve_alert_id), headers=self.headers)
self.assertEqual(response.status_code, 200)
self.assertEqual(db.get_alert(trigger_alert_id).status, "ack")
self.assertEqual(db.get_alert(resolve_alert_id).status, "closed")
def get_alert(id):
try:
alert = db.get_alert(id=id)
except Exception as e:
return jsonify(status="error", message=str(e)), 500
if alert:
body = alert.get_body()
body['href'] = request.base_url
return jsonify(status="ok", total=1, alert=body)
else:
return jsonify(status="error", message="not found", total=0, alert=None), 404
def get_alert(id):
customer = g.get('customer', None)
try:
alert = db.get_alert(id=id, customer=customer)
except Exception as e:
return jsonify(status="error", message=str(e)), 500
if alert:
body = alert.get_body()
body['href'] = absolute_url('/alert/' + alert.id)
return jsonify(status="ok", total=1, alert=body)
else:
return jsonify(status="error", message="not found", total=0, alert=None), 404
def get_alert(id):
customer = g.get('customer', None)
try:
alert = db.get_alert(id=id, customer=customer)
except Exception as e:
return jsonify(status="error", message=str(e)), 500
if alert:
body = alert.get_body()
body['href'] = absolute_url('/alert/' + alert.id)
return jsonify(status="ok", total=1, alert=body)
else:
return jsonify(status="error", message="not found", total=0, alert=None), 404
def get_alert(id):
try:
alert = db.get_alert(id=id)
except Exception as e:
return jsonify(status="error", message=str(e)), 500
if alert:
if g.get('role', None) != 'admin' and not alert.customer == g.get('customer', None):
return jsonify(status="error", message="not found", total=0, alert=None), 404
body = alert.get_body()
body['href'] = request.base_url
return jsonify(status="ok", total=1, alert=body)
else:
return jsonify(status="error", message="not found", total=0, alert=None), 404
def get_alert(id):
try:
alert = db.get_alert(id=id)
except Exception as e:
return jsonify(status="error", message=str(e)), 500
if alert:
if g.get('role', None) != 'admin' and not alert.customer == g.get('customer', None):
return jsonify(status="error", message="not found", total=0, alert=None), 404
body = alert.get_body()
body['href'] = request.base_url
return jsonify(status="ok", total=1, alert=body)
else:
return jsonify(status="error", message="not found", total=0, alert=None), 404
def get_alert(id):
try:
alert = db.get_alert(id=id)
except Exception as e:
return jsonify(status="error", message=str(e)), 500
if alert:
body = alert.get_body()
body['href'] = request.base_url
return jsonify(status="ok", total=1, alert=body)
else:
return jsonify(status="error",
message="not found",
total=0,
alert=None), 404
def set_status(id):
status_started = status_timer.start_timer()
customer = g.get('customer', None)
try:
alert = db.get_alert(id=id, customer=customer)
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
status_timer.stop_timer(status_started)
return jsonify(status="error",
message="not found",
total=0,
alert=None), 404
status = request.json.get('status', None)
text = request.json.get('text', '')
if not status:
status_timer.stop_timer(status_started)
return jsonify(status="error",
message="must supply 'status' as parameter"), 400
try:
process_status(alert, status, text)
except RejectException as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 403
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
try:
alert = db.set_status(id=id, status=status, text=text)
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
if alert:
status_timer.stop_timer(status_started)
return jsonify(status="ok")
else:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="not found"), 404
def set_status(id):
status_started = status_timer.start_timer()
customer = g.get('customer', None)
try:
alert = db.get_alert(id=id, customer=customer)
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="not found", total=0, alert=None), 404
status = request.json.get('status', None)
text = request.json.get('text', '')
if not status:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="must supply 'status' as parameter"), 400
try:
process_status(alert, status, text)
except RejectException as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 403
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
try:
alert = db.set_status(id=id, status=status, text=text)
except Exception as e:
status_timer.stop_timer(status_started)
return jsonify(status="error", message=str(e)), 500
if alert:
status_timer.stop_timer(status_started)
return jsonify(status="ok")
else:
status_timer.stop_timer(status_started)
return jsonify(status="error", message="not found"), 404
def pagerduty():
hook_started = webhook_timer.start_timer()
data = request.json
updated = False
if data and 'messages' in data:
for message in data['messages']:
try:
incident_key, status, text = parse_pagerduty(message)
except ValueError as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 400
customer = g.get('customer', None)
try:
alert = db.get_alert(id=incident_key, customer=customer)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
webhook_timer.stop_timer(hook_started)
return jsonify(stats="error", message="not found"), 404
try:
updated = db.set_status(id=alert.id, status=status, text=text)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
else:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error",
message="no messages in PagerDuty data payload"), 400
webhook_timer.stop_timer(hook_started)
if updated:
return jsonify(status="ok"), 200
else:
return jsonify(status="error",
message="update PagerDuty incident status failed"), 500
def parse_slack(data):
payload = json.loads(data['payload'])
user = payload.get('user', {}).get('name')
alert_key = payload.get('callback_id')
action = payload.get('actions', [{}])[0].get('value')
try:
alert = db.get_alert(id=alert_key)
except Exception as e:
LOG.error(u'User {} is doing a non existent action {}'.format(
user, action))
if not alert:
raise ValueError(u'Alert {} not match'.format(alert_key))
elif not user:
raise ValueError(u'User {} not exist'.format(user))
elif not action:
raise ValueError(u'Non existent action {}'.format(action))
return alert, user, action
def pagerduty():
hook_started = webhook_timer.start_timer()
data = request.json
updated = False
if data and 'messages' in data:
for message in data['messages']:
try:
incident_key, status, text = parse_pagerduty(message)
except ValueError as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 400
customer = g.get('customer', None)
try:
alert = db.get_alert(id=incident_key, customer=customer)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
if not alert:
webhook_timer.stop_timer(hook_started)
return jsonify(stats="error", message="not found"), 404
try:
updated = db.set_status(id=alert.id, status=status, text=text)
except Exception as e:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message=str(e)), 500
else:
webhook_timer.stop_timer(hook_started)
return jsonify(status="error", message="no messages in PagerDuty data payload"), 400
webhook_timer.stop_timer(hook_started)
if updated:
return jsonify(status="ok"), 200
else:
return jsonify(status="error", message="update PagerDuty incident status failed"), 500
def pagerduty():
if not request.json or not 'messages' in request.json:
abort(400)
for message in request.json['messages']:
LOG.debug('%s', json.dumps(message))
id = message['data']['incident']['incident_key']
html_url = message['data']['incident']['html_url']
incident_number = message['data']['incident']['incident_number']
incident_url = '<a href="%s">#%s</a>' % (html_url, incident_number)
LOG.info('PagerDuty incident #%s webhook for alert %s', incident_number, id)
LOG.error('previous status %s', db.get_alert(id=id).status)
if message['type'] == 'incident.trigger':
status = status_code.OPEN
user = message['data']['incident']['assigned_to_user']['name']
text = 'Incident %s assigned to %s' % (incident_url, user)
elif message['type'] == 'incident.acknowledge':
status = status_code.ACK
user = message['data']['incident']['assigned_to_user']['name']
text = 'Incident %s acknowledged by %s' % (incident_url, user)
elif message['type'] == 'incident.unacknowledge':
status = status_code.OPEN
text = 'Incident %s unacknowledged due to timeout' % incident_url
elif message['type'] == 'incident.resolve':
status = status_code.CLOSED
if message['data']['incident']['resolved_by_user']:
user = message['data']['incident']['resolved_by_user']['name']
else:
user = '******'
text = 'Incident %s resolved by %s' % (incident_url, user)
elif message['type'] == 'incident.assign':
status = status_code.ASSIGN
user = message['data']['incident']['assigned_to_user']['name']
text = 'Incident %s manually assigned to %s' % (incident_url, user)
elif message['type'] == 'incident.escalate':
status = status_code.OPEN
user = message['data']['incident']['assigned_to_user']['name']
text = 'Incident %s escalated to %s' % (incident_url, user)
elif message['type'] == 'incident.delegate':
status = status_code.OPEN
user = message['data']['incident']['assigned_to_user']['name']
text = 'Incident %s reassigned due to escalation to %s' % (incident_url, user)
else:
status = status_code.UNKNOWN
text = message['type']
LOG.warn('Unknown PagerDuty message type: %s', message)
LOG.info('PagerDuty webhook %s change status to %s', message['type'], status)
pdAlert = db.update_status(id=id, status=status, text=text)
db.tag_alert(id=id, tags='incident=#%s' % incident_number)
LOG.error('returned status %s', pdAlert.status)
LOG.error('current status %s', db.get_alert(id=id).status)
# Forward alert to notify topic and logger queue
if pdAlert:
pdAlert.origin = 'pagerduty/webhook'
notify.send(pdAlert)
return jsonify(status="ok")
def find_by_id(id: str, customers: List[str]=None) -> 'Alert':
return Alert.from_db(db.get_alert(id, customers))
def find_by_id(id, customers=None):
return Alert.from_db(db.get_alert(id, customers))
def send_message_reply(alert_id, action, user, data):
try:
import telepot
except ImportError as e:
LOG.warning(
"You have configured Telegram but 'telepot' client is not installed",
exc_info=True)
return
try:
bot = telepot.Bot(app.config.get('TELEGRAM_TOKEN'))
dashboard_url = app.config.get('DASHBOARD_URL')
# message info
message_id = data['callback_query']['message']['message_id']
message_log = "\n".join(
data['callback_query']['message']['text'].split('\n')[1:])
# process buttons for reply text
alert = db.get_alert(alert_id)
inline_keyboard, reply = [], "The status of alert {alert} is *{status}* now!"
actions = ['watch', 'unwatch']
if action in actions:
reply = "User `{user}` is _{status}ing_ alert {alert}"
next_action = actions[(actions.index(action) + 1) % len(actions)]
inline_keyboard = [[
{
'text': next_action.capitalize(),
'callback_data': "/{} {}".format(next_action, alert_id)
},
{
'text': 'Ack',
'callback_data': "{} {}".format('/ack', alert_id)
},
{
'text': 'Close',
'callback_data': "{} {}".format('/close', alert_id)
},
]]
# format message response
alert_short_id = alert.get_id(short=True)
alert_url = "{}/#/alert/{}".format(dashboard_url, alert.id)
reply = reply.format(alert=alert_short_id, status=action, user=user)
message = "{alert} *{level} - {event} on {resouce}*\n{log}\n{reply}".format(
alert="[{}]({})".format(alert_short_id, alert_url),
level=alert.severity.capitalize(),
event=alert.event,
resouce=alert.resource,
log=message_log,
reply=reply)
# send message
bot.editMessageText(msg_identifier=(app.config.get('TELEGRAM_CHAT_ID'),
message_id),
text=message,
parse_mode='Markdown',
reply_markup={'inline_keyboard': inline_keyboard})
except Exception as e:
LOG.warning("Error sending reply message", exc_info=True)
def find_by_id(id: str, customers: List[str] = None) -> 'Alert':
return Alert.from_db(db.get_alert(id, customers))
def find_by_id(id, customer=None):
return Alert.from_db(db.get_alert(id, customer))
