def notify(cls, alert, *args, **kwargs):
url = kwargs.get('webhook',
cls._config.get('notifications.slack.webhook'))
repeat = kwargs.get('repeat', 0)
if not url:
raise NotificationError('Webhook is required!')
message = {
'username': '******',
'channel': kwargs.get('channel', '#general'),
'text': kwargs.get('message', cls._get_subject(alert)),
'icon_emoji': ':bar_chart:',
}
headers = {
'User-agent': get_user_agent(),
'Content-type': 'application/json',
}
try:
logger.info('Sending to %s %s', url, message)
r = requests.post(url, json=message, headers=headers, timeout=5)
r.raise_for_status()
except:
logger.exception('Slack notification failed!')
return repeat
def notify(cls, alert, *args, **kwargs):
current_span = extract_span_from_kwargs(**kwargs)
alert_def = alert['alert_def']
current_span.set_tag('alert_id', alert_def['id'])
entity = alert.get('entity')
is_changed = alert.get('alert_changed', False)
is_alert = alert.get('is_alert', False)
current_span.set_tag('entity', entity['id'])
current_span.set_tag('alert_changed', bool(is_changed))
current_span.set_tag('is_alert', is_alert)
url = kwargs.get('webhook',
cls._config.get('notifications.slack.webhook'))
repeat = kwargs.get('repeat', 0)
current_span.log_kv({'channel': kwargs.get('channel')})
if not url:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Missing webhook!'})
raise NotificationError('Webhook is required!')
message = {
'username': '******',
'channel': kwargs.get('channel', '#general'),
'text': kwargs.get('message', cls._get_subject(alert)),
'icon_emoji': ':bar_chart:',
}
headers = {
'User-agent': get_user_agent(),
'Content-type': 'application/json',
}
try:
logger.info('Sending to %s %s', url, message)
r = requests.post(url, json=message, headers=headers, timeout=5)
r.raise_for_status()
except Exception as e:
current_span.set_tag('error', True)
current_span.log_kv({'exception': str(e)})
logger.exception('Slack notification failed!')
return repeat
def notify(cls, alert, per_entity=False, include_alert=True, message='', repeat=0, **kwargs):
url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'
repeat = kwargs.get('repeat', 0)
# Auth key!
service_key = kwargs.get('service_key', cls._config.get('notifications.pagerduty.servicekey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
if not service_key:
raise NotificationError('Service key is required!')
entity = alert.get('entity')
is_alert = alert.get('is_alert')
event_type = 'trigger' if is_alert else 'resolve'
alert_id = alert['alert_def']['id']
key = 'ZMON-{}'.format(alert_id) if not per_entity else 'ZMON-{}-{}'.format(alert_id, entity['id'])
description = message if message else cls._get_subject(alert)
message = {
'service_key': service_key,
'event_type': event_type,
'incident_key': key,
'description': description,
'client': 'ZMON',
'client_url': urlparse.urljoin(zmon_host, '/#/alert-details/{}'.format(alert_id)) if zmon_host else '',
'details': json.dumps(alert, cls=JsonDataEncoder) if include_alert else '',
}
try:
logger.info('Sending to %s %s', url, message)
headers = {'User-Agent': get_user_agent(), 'Content-type': 'application/json'}
r = requests.post(url, json=message, headers=headers, timeout=5)
r.raise_for_status()
except Exception as ex:
logger.exception('Notifying Pagerduty failed %s', ex)
return repeat
def notify(cls,
alert,
teams=None,
per_entity=False,
include_alert=True,
priority=None,
message='',
description='',
custom_fields=None,
**kwargs):
current_span = extract_span_from_kwargs(**kwargs)
url = 'https://api.opsgenie.com/v2/alerts'
repeat = kwargs.get('repeat', 0)
# Auth key!
api_key = kwargs.get('api_key',
cls._config.get('notifications.opsgenie.apikey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
entity = alert.get('entity')
is_changed = alert.get('alert_changed', False)
is_alert = alert.get('is_alert', False)
current_span.set_tag('entity', entity['id'])
current_span.set_tag('alert_changed', bool(is_changed))
current_span.set_tag('is_alert', is_alert)
alert_def = alert['alert_def']
current_span.set_tag('alert_id', alert_def['id'])
if not api_key:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'API key is required!'})
raise NotificationError('API key is required!')
if not isinstance(teams, (list, basestring)):
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Missing team!'})
raise NotificationError(
'Missing "teams" parameter. Either a team name or list of team names is required.'
)
current_span.log_kv({'teams': teams})
if priority and priority not in PRIORITIES:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Invalid priorities'})
raise NotificationError(
'Invalid priority. Valid values are: {}'.format(PRIORITIES))
if teams and isinstance(teams, basestring):
teams = [{'name': teams}]
else:
teams = [{'name': t} for t in teams]
if not is_changed and not per_entity:
return repeat
alert_id = alert['alert_def']['id']
alias = 'ZMON-{}'.format(
alert_id) if not per_entity else 'ZMON-{}-{}'.format(
alert_id, entity['id'])
note = alert_url = urlparse.urljoin(
zmon_host,
'/#/alert-details/{}'.format(alert_id)) if zmon_host else ''
if not priority:
priority = 'P1' if int(
alert['alert_def']['priority']) == 1 else 'P3'
responsible_team = alert['alert_def'].get('responsible_team',
teams[0]['name'])
msg = message if message else cls._get_subject(alert,
include_event=False)
details = {
'alert_evaluation_ts': alert.get('alert_evaluation_ts',
time.time())
}
alert_details = {
'worker':
alert['worker'],
'zmon_team':
alert['alert_def']['team'],
'entity':
entity['id'],
'infrastructure_account':
entity.get('infrastructure_account', 'UNKNOWN'),
'alert_url':
alert_url,
}
params = {}
if is_alert:
tags = alert['alert_def'].get('tags', [])
tags.append(alert['alert_def']['id'])
data = {
'alias': alias,
'teams': teams,
'message': '[{}] - {}'.format(
responsible_team,
msg), # TODO: remove when it is no longer needed!
'source': alert.get('worker', ''),
'description': description,
'entity': entity['id'],
'note': note,
'priority': priority,
'tags': tags,
'details': details,
}
if isinstance(custom_fields, dict):
data['details'].update(custom_fields)
if include_alert:
data['details'].update(alert_details)
else:
logger.info('Closing Opsgenie alert {}'.format(alias))
url = 'https://api.opsgenie.com/v2/alerts/{}/close'.format(alias)
data = {
'user': '******',
'source': alert.get('worker', 'ZMON Worker'),
'note': note,
}
params = {'identifierType': 'alias'}
try:
logger.info('Notifying Opsgenie %s %s', url, message)
headers = {
'User-Agent': get_user_agent(),
'Content-type': 'application/json',
'Authorization': 'GenieKey {}'.format(api_key),
}
r = requests.post(url,
data=json.dumps(data,
cls=JsonDataEncoder,
sort_keys=True),
headers=headers,
timeout=5,
params=params)
r.raise_for_status()
except requests.HTTPError as e:
current_span.set_tag('error', True)
logger.error('HTTP Error ({}) {}'.format(e.response.status_code,
e.response.text))
except Exception:
current_span.set_tag('error', True)
current_span.log_kv({'exception': traceback.format_exc()})
logger.exception('Notifying Opsgenie failed')
return repeat
def notify(cls, alert, *args, **kwargs):
current_span = extract_span_from_kwargs(**kwargs)
repeat = kwargs.get('repeat', 0)
alert_def = alert['alert_def']
per_entity = kwargs.get('per_entity', True)
current_span.set_tag('alert_id', alert_def['id'])
entity = alert.get('entity', {})
is_changed = alert.get('alert_changed', False)
is_alert = alert.get('is_alert', False)
current_span.set_tag('entity', entity.get('id'))
current_span.set_tag('alert_changed', bool(is_changed))
current_span.set_tag('is_alert', is_alert)
if not cls._config.get('notifications.mail.on', True):
current_span.set_tag('mail_enabled', False)
logger.info('Not sending email for alert: {}. Mail notification is not enabled.'.format(alert_def['id']))
return repeat
if not is_changed and not per_entity:
return repeat
sender = cls._config.get('notifications.mail.sender')
subject = cls._get_subject(alert, custom_message=kwargs.get('subject'))
html = kwargs.get('html', False)
cc = kwargs.get('cc', [])
if type(cc) is not list:
cc = [cc]
hide_recipients = kwargs.get('hide_recipients', True)
include_value = kwargs.get('include_value', True)
include_definition = kwargs.get('include_definition', True)
include_captures = kwargs.get('include_captures', True)
include_entity = kwargs.get('include_entity', True)
expanded_alert_name = cls._get_expanded_alert_name(alert)
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
alert_url = urlparse.urljoin(zmon_host, '/#/alert-details/{}'.format(alert_def['id'])) if zmon_host else ''
try:
tmpl = jinja_env.get_template('alert.txt')
body_plain = tmpl.render(expanded_alert_name=expanded_alert_name,
include_value=include_value,
include_definition=include_definition,
include_captures=include_captures,
include_entity=include_entity,
alert_url=alert_url,
**alert)
except Exception:
current_span.set_tag('error', True)
current_span.log_kv({'exception': traceback.format_exc()})
logger.exception('Error parsing email template for alert %s with id %s', alert_def['name'], alert_def['id'])
else:
if html:
current_span.set_tag('html', True)
msg = MIMEMultipart('alternative')
tmpl = jinja_env.get_template('alert.html')
body_html = tmpl.render(expanded_alert_name=expanded_alert_name,
include_value=include_value,
include_definition=include_definition,
include_captures=include_captures,
include_entity=include_entity,
alert_url=alert_url,
**alert)
part1 = MIMEText(body_plain.encode('utf-8'), 'plain', 'utf-8')
part2 = MIMEText(body_html.encode('utf-8'), 'html', 'utf-8')
msg.attach(part1)
msg.attach(part2)
else:
msg = MIMEText(body_plain.encode('utf-8'), 'plain', 'utf-8')
msg['Subject'] = subject
msg['From'] = 'ZMON 2 <{}>'.format(sender)
args = BaseNotification.resolve_group(args)
if hide_recipients:
msg['To'] = 'Undisclosed Recipients <{}>'.format(sender)
msg['Bcc'] = ', '.join(args)
else:
msg['To'] = ', '.join(args)
msg['Cc'] = ', '.join(cc)
mail_host = cls._config.get('notifications.mail.host', 'localhost')
mail_port = cls._config.get('notifications.mail.port', '25')
try:
if mail_host != 'localhost':
if cls._config.get('notifications.mail.tls', False):
logger.info('Mail notification using TLS!')
current_span.set_tag('tls', True)
s = smtplib.SMTP(mail_host, mail_port)
s.ehlo()
if not s.has_extn('STARTTLS'):
raise NotificationError('Mail server ({}) does not support TLS!'.format(mail_host))
s.starttls()
s.ehlo()
else:
current_span.set_tag('tls', False)
s = smtplib.SMTP_SSL(mail_host, mail_port)
else:
s = smtplib.SMTP(mail_host, mail_port)
except Exception:
current_span.set_tag('error', True)
logger.exception('Error connecting to SMTP server %s for alert %s with id %s',
mail_host, alert_def['name'], alert_def['id'])
else:
try:
mail_user = cls._config.get('notifications.mail.user', None)
if mail_user is not None:
s.login(mail_user, cls._config.get('notifications.mail.password'))
s.sendmail(sender, list(args) + cc, msg.as_string())
except SMTPAuthenticationError:
logger.exception(
'Error sending email for alert %s with id %s: authentication failed for %s',
alert_def['name'], alert_def['id'], mail_user)
except Exception:
current_span.set_tag('error', True)
current_span.log_kv({'exception': traceback.format_exc()})
logger.exception(
'Error sending email for alert %s with id %s', alert_def['name'], alert_def['id'])
finally:
s.quit()
finally:
return repeat
def notify(cls,
alert,
url=None,
body=None,
params=None,
headers=None,
timeout=5,
oauth2=False,
include_alert=True,
repeat=0):
urls = cls._config.get('notifications.http.whitelist.urls', [])
allow_any = cls._config.get('notifications.http.allow.all', False)
default_url = cls._config.get('notifications.http.default.url', None)
if isinstance(urls, basestring):
urls = urls.replace(' ', '').split(',')
if not url and not default_url:
raise NotificationError('URL is required!')
if not url:
url = default_url
elif not allow_any and url not in urls:
raise NotificationError(
'URL "{}" is not allowed. Please check worker white list URLs.'
.format(url))
if not is_absolute_http_url(url):
raise NotificationError('Absolute URL is required!')
# HTTP headers.
if not headers:
headers = {}
default_headers = cls._config.get('notifications.http.headers', {})
default_headers.update(headers)
if oauth2:
headers.update(
{'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
headers['User-Agent'] = get_user_agent()
if include_alert:
data = {
'alert': alert,
'body': body,
}
else:
data = body
try:
logger.info('Sending HTTP POST request to {}'.format(url))
r = requests.post(url,
data=json.dumps(data, cls=JsonDataEncoder),
params=params,
headers=headers,
timeout=timeout)
r.raise_for_status()
except:
logger.exception('Request failed!')
return repeat
is_protected = True # localhost is fine
s = smtplib.SMTP(mail_host, mail_port)
except Exception, e:
current_span.set_tag('error', True)
logger.exception(
'Error connecting to SMTP server %s for alert %s with id %s: %s',
mail_host, alert_def['name'], alert_def['id'], str(e))
else:
try:
mail_user = cls._config.get('notifications.mail.user',
None)
if mail_user:
if not is_protected:
raise NotificationError(
'Mail server ({}) does not support TLS / STARTTLS!'
.format(mail_host))
s.login(mail_user,
cls._config.get('notifications.mail.password'))
s.sendmail(sender, list(args) + cc, msg.as_string())
except SMTPAuthenticationError:
logger.exception(
'Error sending email for alert %s with id %s: authentication failed for %s',
alert_def['name'], alert_def['id'], mail_user)
except Exception, e:
current_span.set_tag('error', True)
current_span.log_kv({'exception': traceback.format_exc()})
logger.exception(
'Error sending email for alert %s with id %s: %s',
alert_def['name'], alert_def['id'], str(e))
def notify(cls,
alert,
per_entity=False,
include_alert=True,
message='',
repeat=0,
**kwargs):
url = 'https://events.pagerduty.com/v2/enqueue'
repeat = kwargs.get('repeat', 0)
# Auth key!
routing_key = kwargs.get(
'routing_key',
cls._config.get('notifications.pagerduty.servicekey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
if not routing_key:
raise NotificationError('Service key is required!')
entity = alert.get('entity')
is_changed = alert.get('alert_changed')
is_alert = alert.get('is_alert')
if not is_changed and not per_entity:
return repeat
event_action = 'trigger' if is_alert else 'resolve'
alert_id = alert['alert_def']['id']
key = 'ZMON-{}'.format(
alert_id) if not per_entity else 'ZMON-{}-{}'.format(
alert_id, entity['id'])
description = message if message else cls._get_subject(
alert, include_event=False)
alert_class = kwargs.get('alert_class', '')
alert_group = kwargs.get('alert_group', '')
message = {
'routing_key':
routing_key,
'event_action':
event_action,
'dedup_key':
key,
'client':
'ZMON',
'client_url':
urlparse.urljoin(zmon_host, '/#/alert-details/{}'.format(alert_id))
if zmon_host else '',
'payload': {
'summary':
description,
'source':
alert.get('worker', ''),
'severity':
'critical'
if int(alert['alert_def']['priority']) == 1 else 'error',
'component':
entity['id'],
'custom_details':
alert if include_alert else {},
'class':
alert_class,
'group':
alert_group,
},
}
try:
logger.info('Notifying Pagerduty %s %s', url, message)
headers = {
'User-Agent': get_user_agent(),
'Content-type': 'application/json'
}
r = requests.post(url,
data=json.dumps(message, cls=JsonDataEncoder),
headers=headers,
timeout=5)
r.raise_for_status()
except:
logger.exception('Notifying Pagerduty failed')
return repeat
def notify(cls,
alert,
teams=None,
per_entity=False,
include_alert=True,
message='',
**kwargs):
url = 'https://api.opsgenie.com/v2/alerts'
repeat = kwargs.get('repeat', 0)
# Auth key!
api_key = kwargs.get('api_key',
cls._config.get('notifications.opsgenie.apikey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
if not api_key:
raise NotificationError('API key is required!')
if not isinstance(teams, (list, basestring)):
raise NotificationError(
'Missing "teams" parameter. Either a team name or list of team names is required.'
)
if teams and isinstance(teams, basestring):
teams = [{'name': teams}]
else:
teams = [{'name': t} for t in teams]
entity = alert.get('entity')
is_changed = alert.get('alert_changed')
is_alert = alert.get('is_alert')
if not is_changed and not per_entity:
return repeat
alert_id = alert['alert_def']['id']
alias = 'ZMON-{}'.format(
alert_id) if not per_entity else 'ZMON-{}-{}'.format(
alert_id, entity['id'])
note = urlparse.urljoin(
zmon_host,
'/#/alert-details/{}'.format(alert_id)) if zmon_host else ''
details = {
'worker':
alert['worker'],
'id':
alert_id,
'name':
alert['alert_def']['name'],
'team':
alert['alert_def']['team'],
'responsible_team':
alert['alert_def']['responsible_team'],
'entity':
entity['id'],
'infrastructure_account':
entity.get('infrastructure_account', 'UNKNOWN'),
}
params = {}
if is_alert:
data = {
'alias': alias,
'teams': teams,
'message': message if message else cls._get_subject(alert),
'source': alert.get('worker', ''),
'entity': entity['id'],
'note': note,
'priority':
'P1' if int(alert['alert_def']['priority']) == 1 else 'P3',
'tags': alert['alert_def'].get('tags', [])
}
if include_alert:
data['details'] = details
else:
logger.info('Closing Opsgenie alert {}'.format(alias))
url = 'https://api.opsgenie.com/v2/alerts/{}/close'.format(alias)
data = {
'user': '******',
'source': alert.get('worker', 'ZMON Worker'),
'note': note,
}
params = {'identifierType': 'alias'}
try:
logger.info('Notifying Opsgenie %s %s', url, message)
headers = {
'User-Agent': get_user_agent(),
'Content-type': 'application/json',
'Authorization': 'GenieKey {}'.format(api_key),
}
r = requests.post(url,
data=json.dumps(data,
cls=JsonDataEncoder,
sort_keys=True),
headers=headers,
timeout=5,
params=params)
r.raise_for_status()
except requests.HTTPError as e:
logger.error('HTTP Error ({}) {}'.format(e.response.status_code,
e.response.text))
except:
logger.exception('Notifying Opsgenie failed')
return repeat
def notify(cls,
alert,
per_entity=False,
include_alert=True,
message='',
**kwargs):
url = 'https://api.opsgenie.com/v1/json/alert'
repeat = kwargs.get('repeat', 0)
# Auth key!
api_key = kwargs.get('api_key',
cls._config.get('notifications.opsgenie.apikey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
if not api_key:
raise NotificationError('API key is required!')
entity = alert.get('entity')
is_alert = alert.get('is_alert')
alert_id = alert['alert_def']['id']
alias = 'ZMON-{}'.format(
alert_id) if not per_entity else 'ZMON-{}-{}'.format(
alert_id, entity['id'])
note = urlparse.urljoin(
zmon_host,
'/#/alert-details/{}'.format(alert_id)) if zmon_host else ''
if is_alert:
data = {
'apiKey': api_key,
'alias': alias,
'message': message if message else cls._get_subject(alert),
'source': 'ZMON',
'entity': entity['id'],
'note': note,
'details': alert if include_alert else {},
}
else:
logger.info('Closing Opsgenie alert {}'.format(alias))
url = 'https://api.opsgenie.com/v1/json/alert/close'
data = {
'apiKey': api_key,
'alias': alias,
'source': 'ZMON',
'note': note,
}
try:
logger.info('Sending to %s %s', url, message)
headers = {
'User-Agent': get_user_agent(),
'Content-type': 'application/json'
}
r = requests.post(url,
data=json.dumps(data,
cls=JsonDataEncoder,
sort_keys=True),
headers=headers,
timeout=5)
r.raise_for_status()
except requests.HTTPError as e:
logger.error('HTTP Error ({}) {}'.format(e.response.status_code,
e.response.text))
except:
logger.exception('Notifying Opsgenie failed')
return repeat
def notify(cls,
alert,
url=None,
body=None,
params=None,
headers=None,
timeout=5,
oauth2=False,
include_alert=True,
repeat=0,
**kwargs):
current_span = extract_span_from_kwargs(**kwargs)
urls = cls._config.get('notifications.http.whitelist.urls', [])
allow_any = cls._config.get('notifications.http.allow.all', False)
default_url = cls._config.get('notifications.http.default.url', None)
alert_def = alert['alert_def']
current_span.set_tag('alert_id', alert_def['id'])
entity = alert.get('entity', {})
is_changed = alert.get('alert_changed', False)
is_alert = alert.get('is_alert', False)
current_span.set_tag('entity', entity.get('id'))
current_span.set_tag('alert_changed', bool(is_changed))
current_span.set_tag('is_alert', is_alert)
if isinstance(urls, basestring):
urls = urls.replace(' ', '').split(',')
if not url and not default_url:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Missing URL!'})
raise NotificationError('URL is required!')
if not url:
url = default_url
elif not allow_any and url not in urls:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'URL is not in whitelist'})
raise NotificationError(
'URL "{}" is not allowed. Please check worker white list URLs.'
.format(url))
if not is_absolute_http_url(url):
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Absolute URL required!'})
raise NotificationError('Absolute URL is required!')
# HTTP headers.
if not headers:
headers = {}
default_headers = cls._config.get('notifications.http.headers', {})
default_headers.update(headers)
if oauth2:
headers.update(
{'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
headers['User-Agent'] = get_user_agent()
if include_alert:
data = {
'alert': alert,
'body': body,
}
else:
data = body
try:
logger.info('Sending HTTP POST request to {}'.format(url))
r = requests.post(url,
data=json.dumps(data, cls=JsonDataEncoder),
params=params,
headers=headers,
timeout=timeout)
r.raise_for_status()
except Exception:
current_span.set_tag('error', True)
logger.exception('Request failed!')
return repeat