def notify(cls, alert, *args, **kwargs): url = kwargs.get('webhook', cls._config.get('notifications.slack.webhook')) repeat = kwargs.get('repeat', 0) if not url: raise NotificationError('Webhook is required!') message = { 'username': '******', 'channel': kwargs.get('channel', '#general'), 'text': kwargs.get('message', cls._get_subject(alert)), 'icon_emoji': ':bar_chart:', } headers = { 'User-agent': get_user_agent(), 'Content-type': 'application/json', } try: logger.info('Sending to %s %s', url, message) r = requests.post(url, json=message, headers=headers, timeout=5) r.raise_for_status() except: logger.exception('Slack notification failed!') return repeat
def notify(cls, alert, *args, **kwargs): current_span = extract_span_from_kwargs(**kwargs) alert_def = alert['alert_def'] current_span.set_tag('alert_id', alert_def['id']) entity = alert.get('entity') is_changed = alert.get('alert_changed', False) is_alert = alert.get('is_alert', False) current_span.set_tag('entity', entity['id']) current_span.set_tag('alert_changed', bool(is_changed)) current_span.set_tag('is_alert', is_alert) url = kwargs.get('webhook', cls._config.get('notifications.slack.webhook')) repeat = kwargs.get('repeat', 0) current_span.log_kv({'channel': kwargs.get('channel')}) if not url: current_span.set_tag('notification_invalid', True) current_span.log_kv({'reason': 'Missing webhook!'}) raise NotificationError('Webhook is required!') message = { 'username': '******', 'channel': kwargs.get('channel', '#general'), 'text': kwargs.get('message', cls._get_subject(alert)), 'icon_emoji': ':bar_chart:', } headers = { 'User-agent': get_user_agent(), 'Content-type': 'application/json', } try: logger.info('Sending to %s %s', url, message) r = requests.post(url, json=message, headers=headers, timeout=5) r.raise_for_status() except Exception as e: current_span.set_tag('error', True) current_span.log_kv({'exception': str(e)}) logger.exception('Slack notification failed!') return repeat
def notify(cls, alert, per_entity=False, include_alert=True, message='', repeat=0, **kwargs): url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json' repeat = kwargs.get('repeat', 0) # Auth key! service_key = kwargs.get('service_key', cls._config.get('notifications.pagerduty.servicekey')) zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host')) if not service_key: raise NotificationError('Service key is required!') entity = alert.get('entity') is_alert = alert.get('is_alert') event_type = 'trigger' if is_alert else 'resolve' alert_id = alert['alert_def']['id'] key = 'ZMON-{}'.format(alert_id) if not per_entity else 'ZMON-{}-{}'.format(alert_id, entity['id']) description = message if message else cls._get_subject(alert) message = { 'service_key': service_key, 'event_type': event_type, 'incident_key': key, 'description': description, 'client': 'ZMON', 'client_url': urlparse.urljoin(zmon_host, '/#/alert-details/{}'.format(alert_id)) if zmon_host else '', 'details': json.dumps(alert, cls=JsonDataEncoder) if include_alert else '', } try: logger.info('Sending to %s %s', url, message) headers = {'User-Agent': get_user_agent(), 'Content-type': 'application/json'} r = requests.post(url, json=message, headers=headers, timeout=5) r.raise_for_status() except Exception as ex: logger.exception('Notifying Pagerduty failed %s', ex) return repeat
def notify(cls, alert, teams=None, per_entity=False, include_alert=True, priority=None, message='', description='', custom_fields=None, **kwargs): current_span = extract_span_from_kwargs(**kwargs) url = 'https://api.opsgenie.com/v2/alerts' repeat = kwargs.get('repeat', 0) # Auth key! api_key = kwargs.get('api_key', cls._config.get('notifications.opsgenie.apikey')) zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host')) entity = alert.get('entity') is_changed = alert.get('alert_changed', False) is_alert = alert.get('is_alert', False) current_span.set_tag('entity', entity['id']) current_span.set_tag('alert_changed', bool(is_changed)) current_span.set_tag('is_alert', is_alert) alert_def = alert['alert_def'] current_span.set_tag('alert_id', alert_def['id']) if not api_key: current_span.set_tag('notification_invalid', True) current_span.log_kv({'reason': 'API key is required!'}) raise NotificationError('API key is required!') if not isinstance(teams, (list, basestring)): current_span.set_tag('notification_invalid', True) current_span.log_kv({'reason': 'Missing team!'}) raise NotificationError( 'Missing "teams" parameter. Either a team name or list of team names is required.' ) current_span.log_kv({'teams': teams}) if priority and priority not in PRIORITIES: current_span.set_tag('notification_invalid', True) current_span.log_kv({'reason': 'Invalid priorities'}) raise NotificationError( 'Invalid priority. Valid values are: {}'.format(PRIORITIES)) if teams and isinstance(teams, basestring): teams = [{'name': teams}] else: teams = [{'name': t} for t in teams] if not is_changed and not per_entity: return repeat alert_id = alert['alert_def']['id'] alias = 'ZMON-{}'.format( alert_id) if not per_entity else 'ZMON-{}-{}'.format( alert_id, entity['id']) note = alert_url = urlparse.urljoin( zmon_host, '/#/alert-details/{}'.format(alert_id)) if zmon_host else '' if not priority: priority = 'P1' if int( alert['alert_def']['priority']) == 1 else 'P3' responsible_team = alert['alert_def'].get('responsible_team', teams[0]['name']) msg = message if message else cls._get_subject(alert, include_event=False) details = { 'alert_evaluation_ts': alert.get('alert_evaluation_ts', time.time()) } alert_details = { 'worker': alert['worker'], 'zmon_team': alert['alert_def']['team'], 'entity': entity['id'], 'infrastructure_account': entity.get('infrastructure_account', 'UNKNOWN'), 'alert_url': alert_url, } params = {} if is_alert: tags = alert['alert_def'].get('tags', []) tags.append(alert['alert_def']['id']) data = { 'alias': alias, 'teams': teams, 'message': '[{}] - {}'.format( responsible_team, msg), # TODO: remove when it is no longer needed! 'source': alert.get('worker', ''), 'description': description, 'entity': entity['id'], 'note': note, 'priority': priority, 'tags': tags, 'details': details, } if isinstance(custom_fields, dict): data['details'].update(custom_fields) if include_alert: data['details'].update(alert_details) else: logger.info('Closing Opsgenie alert {}'.format(alias)) url = 'https://api.opsgenie.com/v2/alerts/{}/close'.format(alias) data = { 'user': '******', 'source': alert.get('worker', 'ZMON Worker'), 'note': note, } params = {'identifierType': 'alias'} try: logger.info('Notifying Opsgenie %s %s', url, message) headers = { 'User-Agent': get_user_agent(), 'Content-type': 'application/json', 'Authorization': 'GenieKey {}'.format(api_key), } r = requests.post(url, data=json.dumps(data, cls=JsonDataEncoder, sort_keys=True), headers=headers, timeout=5, params=params) r.raise_for_status() except requests.HTTPError as e: current_span.set_tag('error', True) logger.error('HTTP Error ({}) {}'.format(e.response.status_code, e.response.text)) except Exception: current_span.set_tag('error', True) current_span.log_kv({'exception': traceback.format_exc()}) logger.exception('Notifying Opsgenie failed') return repeat
def notify(cls, alert, *args, **kwargs): current_span = extract_span_from_kwargs(**kwargs) repeat = kwargs.get('repeat', 0) alert_def = alert['alert_def'] per_entity = kwargs.get('per_entity', True) current_span.set_tag('alert_id', alert_def['id']) entity = alert.get('entity', {}) is_changed = alert.get('alert_changed', False) is_alert = alert.get('is_alert', False) current_span.set_tag('entity', entity.get('id')) current_span.set_tag('alert_changed', bool(is_changed)) current_span.set_tag('is_alert', is_alert) if not cls._config.get('notifications.mail.on', True): current_span.set_tag('mail_enabled', False) logger.info('Not sending email for alert: {}. Mail notification is not enabled.'.format(alert_def['id'])) return repeat if not is_changed and not per_entity: return repeat sender = cls._config.get('notifications.mail.sender') subject = cls._get_subject(alert, custom_message=kwargs.get('subject')) html = kwargs.get('html', False) cc = kwargs.get('cc', []) if type(cc) is not list: cc = [cc] hide_recipients = kwargs.get('hide_recipients', True) include_value = kwargs.get('include_value', True) include_definition = kwargs.get('include_definition', True) include_captures = kwargs.get('include_captures', True) include_entity = kwargs.get('include_entity', True) expanded_alert_name = cls._get_expanded_alert_name(alert) zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host')) alert_url = urlparse.urljoin(zmon_host, '/#/alert-details/{}'.format(alert_def['id'])) if zmon_host else '' try: tmpl = jinja_env.get_template('alert.txt') body_plain = tmpl.render(expanded_alert_name=expanded_alert_name, include_value=include_value, include_definition=include_definition, include_captures=include_captures, include_entity=include_entity, alert_url=alert_url, **alert) except Exception: current_span.set_tag('error', True) current_span.log_kv({'exception': traceback.format_exc()}) logger.exception('Error parsing email template for alert %s with id %s', alert_def['name'], alert_def['id']) else: if html: current_span.set_tag('html', True) msg = MIMEMultipart('alternative') tmpl = jinja_env.get_template('alert.html') body_html = tmpl.render(expanded_alert_name=expanded_alert_name, include_value=include_value, include_definition=include_definition, include_captures=include_captures, include_entity=include_entity, alert_url=alert_url, **alert) part1 = MIMEText(body_plain.encode('utf-8'), 'plain', 'utf-8') part2 = MIMEText(body_html.encode('utf-8'), 'html', 'utf-8') msg.attach(part1) msg.attach(part2) else: msg = MIMEText(body_plain.encode('utf-8'), 'plain', 'utf-8') msg['Subject'] = subject msg['From'] = 'ZMON 2 <{}>'.format(sender) args = BaseNotification.resolve_group(args) if hide_recipients: msg['To'] = 'Undisclosed Recipients <{}>'.format(sender) msg['Bcc'] = ', '.join(args) else: msg['To'] = ', '.join(args) msg['Cc'] = ', '.join(cc) mail_host = cls._config.get('notifications.mail.host', 'localhost') mail_port = cls._config.get('notifications.mail.port', '25') try: if mail_host != 'localhost': if cls._config.get('notifications.mail.tls', False): logger.info('Mail notification using TLS!') current_span.set_tag('tls', True) s = smtplib.SMTP(mail_host, mail_port) s.ehlo() if not s.has_extn('STARTTLS'): raise NotificationError('Mail server ({}) does not support TLS!'.format(mail_host)) s.starttls() s.ehlo() else: current_span.set_tag('tls', False) s = smtplib.SMTP_SSL(mail_host, mail_port) else: s = smtplib.SMTP(mail_host, mail_port) except Exception: current_span.set_tag('error', True) logger.exception('Error connecting to SMTP server %s for alert %s with id %s', mail_host, alert_def['name'], alert_def['id']) else: try: mail_user = cls._config.get('notifications.mail.user', None) if mail_user is not None: s.login(mail_user, cls._config.get('notifications.mail.password')) s.sendmail(sender, list(args) + cc, msg.as_string()) except SMTPAuthenticationError: logger.exception( 'Error sending email for alert %s with id %s: authentication failed for %s', alert_def['name'], alert_def['id'], mail_user) except Exception: current_span.set_tag('error', True) current_span.log_kv({'exception': traceback.format_exc()}) logger.exception( 'Error sending email for alert %s with id %s', alert_def['name'], alert_def['id']) finally: s.quit() finally: return repeat
def notify(cls, alert, url=None, body=None, params=None, headers=None, timeout=5, oauth2=False, include_alert=True, repeat=0): urls = cls._config.get('notifications.http.whitelist.urls', []) allow_any = cls._config.get('notifications.http.allow.all', False) default_url = cls._config.get('notifications.http.default.url', None) if isinstance(urls, basestring): urls = urls.replace(' ', '').split(',') if not url and not default_url: raise NotificationError('URL is required!') if not url: url = default_url elif not allow_any and url not in urls: raise NotificationError( 'URL "{}" is not allowed. Please check worker white list URLs.' .format(url)) if not is_absolute_http_url(url): raise NotificationError('Absolute URL is required!') # HTTP headers. if not headers: headers = {} default_headers = cls._config.get('notifications.http.headers', {}) default_headers.update(headers) if oauth2: headers.update( {'Authorization': 'Bearer {}'.format(tokens.get('uid'))}) headers['User-Agent'] = get_user_agent() if include_alert: data = { 'alert': alert, 'body': body, } else: data = body try: logger.info('Sending HTTP POST request to {}'.format(url)) r = requests.post(url, data=json.dumps(data, cls=JsonDataEncoder), params=params, headers=headers, timeout=timeout) r.raise_for_status() except: logger.exception('Request failed!') return repeat
is_protected = True # localhost is fine s = smtplib.SMTP(mail_host, mail_port) except Exception, e: current_span.set_tag('error', True) logger.exception( 'Error connecting to SMTP server %s for alert %s with id %s: %s', mail_host, alert_def['name'], alert_def['id'], str(e)) else: try: mail_user = cls._config.get('notifications.mail.user', None) if mail_user: if not is_protected: raise NotificationError( 'Mail server ({}) does not support TLS / STARTTLS!' .format(mail_host)) s.login(mail_user, cls._config.get('notifications.mail.password')) s.sendmail(sender, list(args) + cc, msg.as_string()) except SMTPAuthenticationError: logger.exception( 'Error sending email for alert %s with id %s: authentication failed for %s', alert_def['name'], alert_def['id'], mail_user) except Exception, e: current_span.set_tag('error', True) current_span.log_kv({'exception': traceback.format_exc()}) logger.exception( 'Error sending email for alert %s with id %s: %s', alert_def['name'], alert_def['id'], str(e))
def notify(cls, alert, per_entity=False, include_alert=True, message='', repeat=0, **kwargs): url = 'https://events.pagerduty.com/v2/enqueue' repeat = kwargs.get('repeat', 0) # Auth key! routing_key = kwargs.get( 'routing_key', cls._config.get('notifications.pagerduty.servicekey')) zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host')) if not routing_key: raise NotificationError('Service key is required!') entity = alert.get('entity') is_changed = alert.get('alert_changed') is_alert = alert.get('is_alert') if not is_changed and not per_entity: return repeat event_action = 'trigger' if is_alert else 'resolve' alert_id = alert['alert_def']['id'] key = 'ZMON-{}'.format( alert_id) if not per_entity else 'ZMON-{}-{}'.format( alert_id, entity['id']) description = message if message else cls._get_subject( alert, include_event=False) alert_class = kwargs.get('alert_class', '') alert_group = kwargs.get('alert_group', '') message = { 'routing_key': routing_key, 'event_action': event_action, 'dedup_key': key, 'client': 'ZMON', 'client_url': urlparse.urljoin(zmon_host, '/#/alert-details/{}'.format(alert_id)) if zmon_host else '', 'payload': { 'summary': description, 'source': alert.get('worker', ''), 'severity': 'critical' if int(alert['alert_def']['priority']) == 1 else 'error', 'component': entity['id'], 'custom_details': alert if include_alert else {}, 'class': alert_class, 'group': alert_group, }, } try: logger.info('Notifying Pagerduty %s %s', url, message) headers = { 'User-Agent': get_user_agent(), 'Content-type': 'application/json' } r = requests.post(url, data=json.dumps(message, cls=JsonDataEncoder), headers=headers, timeout=5) r.raise_for_status() except: logger.exception('Notifying Pagerduty failed') return repeat
def notify(cls, alert, teams=None, per_entity=False, include_alert=True, message='', **kwargs): url = 'https://api.opsgenie.com/v2/alerts' repeat = kwargs.get('repeat', 0) # Auth key! api_key = kwargs.get('api_key', cls._config.get('notifications.opsgenie.apikey')) zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host')) if not api_key: raise NotificationError('API key is required!') if not isinstance(teams, (list, basestring)): raise NotificationError( 'Missing "teams" parameter. Either a team name or list of team names is required.' ) if teams and isinstance(teams, basestring): teams = [{'name': teams}] else: teams = [{'name': t} for t in teams] entity = alert.get('entity') is_changed = alert.get('alert_changed') is_alert = alert.get('is_alert') if not is_changed and not per_entity: return repeat alert_id = alert['alert_def']['id'] alias = 'ZMON-{}'.format( alert_id) if not per_entity else 'ZMON-{}-{}'.format( alert_id, entity['id']) note = urlparse.urljoin( zmon_host, '/#/alert-details/{}'.format(alert_id)) if zmon_host else '' details = { 'worker': alert['worker'], 'id': alert_id, 'name': alert['alert_def']['name'], 'team': alert['alert_def']['team'], 'responsible_team': alert['alert_def']['responsible_team'], 'entity': entity['id'], 'infrastructure_account': entity.get('infrastructure_account', 'UNKNOWN'), } params = {} if is_alert: data = { 'alias': alias, 'teams': teams, 'message': message if message else cls._get_subject(alert), 'source': alert.get('worker', ''), 'entity': entity['id'], 'note': note, 'priority': 'P1' if int(alert['alert_def']['priority']) == 1 else 'P3', 'tags': alert['alert_def'].get('tags', []) } if include_alert: data['details'] = details else: logger.info('Closing Opsgenie alert {}'.format(alias)) url = 'https://api.opsgenie.com/v2/alerts/{}/close'.format(alias) data = { 'user': '******', 'source': alert.get('worker', 'ZMON Worker'), 'note': note, } params = {'identifierType': 'alias'} try: logger.info('Notifying Opsgenie %s %s', url, message) headers = { 'User-Agent': get_user_agent(), 'Content-type': 'application/json', 'Authorization': 'GenieKey {}'.format(api_key), } r = requests.post(url, data=json.dumps(data, cls=JsonDataEncoder, sort_keys=True), headers=headers, timeout=5, params=params) r.raise_for_status() except requests.HTTPError as e: logger.error('HTTP Error ({}) {}'.format(e.response.status_code, e.response.text)) except: logger.exception('Notifying Opsgenie failed') return repeat
def notify(cls, alert, per_entity=False, include_alert=True, message='', **kwargs): url = 'https://api.opsgenie.com/v1/json/alert' repeat = kwargs.get('repeat', 0) # Auth key! api_key = kwargs.get('api_key', cls._config.get('notifications.opsgenie.apikey')) zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host')) if not api_key: raise NotificationError('API key is required!') entity = alert.get('entity') is_alert = alert.get('is_alert') alert_id = alert['alert_def']['id'] alias = 'ZMON-{}'.format( alert_id) if not per_entity else 'ZMON-{}-{}'.format( alert_id, entity['id']) note = urlparse.urljoin( zmon_host, '/#/alert-details/{}'.format(alert_id)) if zmon_host else '' if is_alert: data = { 'apiKey': api_key, 'alias': alias, 'message': message if message else cls._get_subject(alert), 'source': 'ZMON', 'entity': entity['id'], 'note': note, 'details': alert if include_alert else {}, } else: logger.info('Closing Opsgenie alert {}'.format(alias)) url = 'https://api.opsgenie.com/v1/json/alert/close' data = { 'apiKey': api_key, 'alias': alias, 'source': 'ZMON', 'note': note, } try: logger.info('Sending to %s %s', url, message) headers = { 'User-Agent': get_user_agent(), 'Content-type': 'application/json' } r = requests.post(url, data=json.dumps(data, cls=JsonDataEncoder, sort_keys=True), headers=headers, timeout=5) r.raise_for_status() except requests.HTTPError as e: logger.error('HTTP Error ({}) {}'.format(e.response.status_code, e.response.text)) except: logger.exception('Notifying Opsgenie failed') return repeat
def notify(cls, alert, url=None, body=None, params=None, headers=None, timeout=5, oauth2=False, include_alert=True, repeat=0, **kwargs): current_span = extract_span_from_kwargs(**kwargs) urls = cls._config.get('notifications.http.whitelist.urls', []) allow_any = cls._config.get('notifications.http.allow.all', False) default_url = cls._config.get('notifications.http.default.url', None) alert_def = alert['alert_def'] current_span.set_tag('alert_id', alert_def['id']) entity = alert.get('entity', {}) is_changed = alert.get('alert_changed', False) is_alert = alert.get('is_alert', False) current_span.set_tag('entity', entity.get('id')) current_span.set_tag('alert_changed', bool(is_changed)) current_span.set_tag('is_alert', is_alert) if isinstance(urls, basestring): urls = urls.replace(' ', '').split(',') if not url and not default_url: current_span.set_tag('notification_invalid', True) current_span.log_kv({'reason': 'Missing URL!'}) raise NotificationError('URL is required!') if not url: url = default_url elif not allow_any and url not in urls: current_span.set_tag('notification_invalid', True) current_span.log_kv({'reason': 'URL is not in whitelist'}) raise NotificationError( 'URL "{}" is not allowed. Please check worker white list URLs.' .format(url)) if not is_absolute_http_url(url): current_span.set_tag('notification_invalid', True) current_span.log_kv({'reason': 'Absolute URL required!'}) raise NotificationError('Absolute URL is required!') # HTTP headers. if not headers: headers = {} default_headers = cls._config.get('notifications.http.headers', {}) default_headers.update(headers) if oauth2: headers.update( {'Authorization': 'Bearer {}'.format(tokens.get('uid'))}) headers['User-Agent'] = get_user_agent() if include_alert: data = { 'alert': alert, 'body': body, } else: data = body try: logger.info('Sending HTTP POST request to {}'.format(url)) r = requests.post(url, data=json.dumps(data, cls=JsonDataEncoder), params=params, headers=headers, timeout=timeout) r.raise_for_status() except Exception: current_span.set_tag('error', True) logger.exception('Request failed!') return repeat