def notify(cls, alert, *args, **kwargs):
url = kwargs.get('webhook',
cls._config.get('notifications.slack.webhook'))
repeat = kwargs.get('repeat', 0)
if not url:
raise NotificationError('Webhook is required!')
message = {
'username': '******',
'channel': kwargs.get('channel', '#general'),
'text': kwargs.get('message', cls._get_subject(alert)),
'icon_emoji': ':bar_chart:',
}
headers = {
'User-agent': get_user_agent(),
'Content-type': 'application/json',
}
try:
logger.info('Sending to %s %s', url, message)
r = requests.post(url, json=message, headers=headers, timeout=5)
r.raise_for_status()
except:
logger.exception('Slack notification failed!')
return repeat
def __request(self, raise_error=True, post_data=None):
if self.__r is None:
if self.max_retries:
s = requests.Session()
s.mount('', HTTPAdapter(max_retries=self.max_retries))
else:
s = requests
base_url = self.url
basic_auth = None
url_parsed = urlparse.urlsplit(base_url)
if url_parsed and url_parsed.username and url_parsed.password:
base_url = base_url.replace(
"{0}:{1}@".format(urllib.quote(url_parsed.username), urllib.quote(url_parsed.password)), "")
base_url = base_url.replace("{0}:{1}@".format(url_parsed.username, url_parsed.password), "")
basic_auth = (url_parsed.username, url_parsed.password)
self.clean_url = base_url
if self.oauth2:
self._headers.update({'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
self._headers.update({'User-Agent': get_user_agent()})
try:
if post_data is None:
self.__r = s.get(base_url, params=self.params, timeout=self.timeout, verify=self.verify,
headers=self._headers, auth=basic_auth)
else:
self.__r = s.post(base_url, params=self.params, timeout=self.timeout, verify=self.verify,
headers=self._headers, auth=basic_auth, data=json.dumps(post_data))
except requests.Timeout, e:
raise HttpError('timeout', self.clean_url), None, sys.exc_info()[2]
except requests.ConnectionError, e:
raise HttpError('connection failed', self.clean_url), None, sys.exc_info()[2]
def get_headers(headers=None):
h = {'User-Agent': get_user_agent()}
if headers:
h.update(headers)
return h
def __init__(self,
url=None,
username=None,
password=None,
es_url=None,
index_prefix=''):
if not url:
raise RuntimeError(
'AppDynamics plugin improperly configured. URL is required!')
self.url = url
self.es_url = es_url
self.index_prefix = index_prefix
self.__oauth2 = False
self.__session = requests.Session()
if not username or not password:
self.__oauth2 = True
self.__session.headers.update(
{'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
else:
self.__session.auth = (username, password)
self.__session.headers.update({'User-Agent': get_user_agent()})
self.__session.params = {'output': 'json'}
self.__session.timeout = 3
def test_http_redirects(monkeypatch, fx_redirects):
kwargs, code = fx_redirects
exp_allow_redirects = False if 'allow_redirects' not in kwargs else kwargs[
'allow_redirects']
resp = MagicMock()
resp.status_code = code
resp.text = ''
redirect_url = 'http://example.org/some-file'
resp.headers = {'Location': redirect_url}
method = MagicMock()
method.return_value = resp
patch = 'requests.{}'.format(kwargs['method'].lower())
monkeypatch.setattr(patch, method)
http = HttpWrapper('http://example.org', **kwargs)
assert code == http.code()
assert '' == http.text()
assert redirect_url == http.headers()['Location']
method.assert_called_once_with('http://example.org',
auth=None,
headers={'User-Agent': get_user_agent()},
params=None,
timeout=10,
verify=True,
allow_redirects=exp_allow_redirects)
def test_http(monkeypatch):
resp = MagicMock()
resp.status_code = 200
resp.text = '"foo"'
resp.content = resp.text
resp.json.return_value = 'foo'
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://example.org')
assert 200 == http.code()
assert '"foo"' == http.text()
assert 'foo' == http.json()
assert 5 == http.content_size()
get.assert_called_once_with('http://example.org',
auth=None,
headers={'User-Agent': get_user_agent()},
params=None,
timeout=10,
verify=True,
allow_redirects=True)
resp.json.side_effect = Exception('JSON fail')
with pytest.raises(HttpError) as ex:
http.json()
assert 'JSON fail' == ex.value.message
def get_headers(headers=None):
h = {'User-Agent': get_user_agent()}
if headers:
h.update(headers)
return h
def __init__(self, url=None, timeout=10, oauth2=False):
if not url:
raise ConfigurationError('Elasticsearch plugin improperly configured. URL is required!')
self.url = url
self.timeout = timeout
self.oauth2 = oauth2
self._headers = {'User-Agent': get_user_agent()}
def __init__(self, url=None, timeout=10, oauth2=False):
if not url:
raise ConfigurationError(
'Elasticsearch plugin improperly configured. URL is required!')
self.url = url
self.timeout = timeout
self.oauth2 = oauth2
self._headers = {'User-Agent': get_user_agent()}
def notify(cls, alert, *args, **kwargs):
current_span = extract_span_from_kwargs(**kwargs)
repeat = kwargs.get('repeat', 0)
oauth2 = kwargs.get('oauth2', True)
headers = {'Content-type': 'application/json'}
timeout = 5
alert_def = alert['alert_def']
current_span.set_tag('alert_id', alert_def['id'])
entity = alert.get('entity')
is_changed = alert.get('alert_changed', False)
is_alert = alert.get('is_alert', False)
current_span.set_tag('entity', entity['id'])
current_span.set_tag('alert_changed', bool(is_changed))
current_span.set_tag('is_alert', is_alert)
url = cls._config.get('notifications.service.url', None)
if not url:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'No notification service url set!'})
logger.error('No notification service url set')
return repeat
url = url + '/api/v1/twilio'
if oauth2:
headers.update({'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
else:
key = kwargs.get('key', cls._config.get('notifications.service.key'))
headers.update({'Authorization': 'Bearer {}'.format(key)})
headers['User-Agent'] = get_user_agent()
data = {
'message': kwargs.get('message', cls._get_subject(alert)),
'escalation_team': kwargs.get('team', alert['alert_def'].get('team', '')),
'numbers': kwargs.get('numbers', []),
'voice': kwargs.get('voice', 'woman'),
'alert_id': alert['alert_def']['id'],
'entity_id': alert['entity']['id'],
'event_type': 'ALERT_ENDED' if alert and not alert.get('is_alert') else 'ALERT_START',
'alert_changed': alert.get('alert_changed', False),
}
try:
logger.info('Sending HTTP POST request to {}'.format(url))
r = requests.post(url, data=json.dumps(data, cls=JsonDataEncoder), headers=headers, timeout=timeout)
r.raise_for_status()
except Exception:
logger.exception('Twilio Request failed!')
return repeat
def notify(cls, alert, *args, **kwargs):
repeat = kwargs.get('repeat', 0)
oauth2 = kwargs.get('oauth2', True)
headers = {'Content-type': 'application/json'}
timeout = 5
url = cls._config.get('notifications.service.url', None)
if not url:
logger.error('No notification service url set')
return repeat
url = url + '/api/v1/twilio'
if oauth2:
headers.update(
{'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
else:
key = kwargs.get('key',
cls._config.get('notifications.service.key'))
headers.update({'Authorization': 'Bearer {}'.format(key)})
headers['User-Agent'] = get_user_agent()
data = {
'message':
kwargs.get('message', cls._get_subject(alert)),
'escalation_team':
kwargs.get('team', alert['alert_def'].get('team', '')),
'numbers':
kwargs.get('numbers', []),
'voice':
kwargs.get('voice', 'woman'),
'alert_id':
alert['alert_def']['id'],
'entity_id':
alert['entity']['id'],
'event_type':
'ALERT_ENDED'
if alert and not alert.get('is_alert') else 'ALERT_START',
'alert_changed':
alert.get('alert_changed', False),
}
try:
logger.info('Sending HTTP POST request to {}'.format(url))
r = requests.post(url,
data=json.dumps(data, cls=JsonDataEncoder),
headers=headers,
timeout=timeout)
r.raise_for_status()
except:
logger.exception('Twilio Request failed!')
return repeat
def __request(self, raise_error=True, post_data=None):
if self.__r is None:
if self.max_retries:
s = requests.Session()
s.mount('', HTTPAdapter(max_retries=self.max_retries))
else:
s = requests
base_url = self.url
basic_auth = None
url_parsed = urlparse.urlsplit(base_url)
if url_parsed and url_parsed.username and url_parsed.password:
base_url = base_url.replace(
"{0}:{1}@".format(urllib.quote(url_parsed.username),
urllib.quote(url_parsed.password)), "")
base_url = base_url.replace(
"{0}:{1}@".format(url_parsed.username,
url_parsed.password), "")
basic_auth = (url_parsed.username, url_parsed.password)
self.clean_url = base_url
if self.oauth2:
self._headers.update({
'Authorization':
'Bearer {}'.format(tokens.get(self.oauth2_token_name))
})
self._headers.update({'User-Agent': get_user_agent()})
try:
if post_data is None:
# GET or HEAD
get_method = getattr(s, self.__method)
self.__r = get_method(base_url,
params=self.params,
timeout=self.timeout,
verify=self.verify,
headers=self._headers,
auth=basic_auth,
allow_redirects=self.allow_redirects)
else:
self.__r = s.post(base_url,
params=self.params,
timeout=self.timeout,
verify=self.verify,
headers=self._headers,
auth=basic_auth,
data=json.dumps(post_data))
except requests.Timeout, e:
raise HttpError('timeout',
self.clean_url), None, sys.exc_info()[2]
except requests.ConnectionError, e:
raise HttpError('connection failed',
self.clean_url), None, sys.exc_info()[2]
def test_oauth2(monkeypatch):
resp = MagicMock()
resp.status_code = 218
resp.text = 'OK'
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
monkeypatch.setattr('tokens.get', lambda x: 'mytok'
if x is 'uid' else 'myothertok')
http = HttpWrapper('http://example.org', oauth2=True, timeout=2)
assert 218 == http.code()
assert 'OK' == http.text()
get.assert_called_with('http://example.org',
auth=None,
headers={
'Authorization': 'Bearer mytok',
'User-Agent': get_user_agent()
},
params=None,
timeout=2,
verify=True,
allow_redirects=True)
http = HttpWrapper('http://example.org',
oauth2=True,
oauth2_token_name='foo',
timeout=2)
assert 218 == http.code()
assert 'OK' == http.text()
get.assert_called_with('http://example.org',
auth=None,
headers={
'Authorization': 'Bearer myothertok',
'User-Agent': get_user_agent()
},
params=None,
timeout=2,
verify=True,
allow_redirects=True)
def __init__(self, service_url, infrastructure_account, verify=True, oauth2=False):
if not service_url:
raise ConfigurationError('EntitiesWrapper improperly configured. URL is missing!')
self.infrastructure_account = infrastructure_account
self.__service_url = urlparse.urljoin(service_url, 'api/v1/')
self.__session = requests.Session()
self.__session.headers.update({'User-Agent': get_user_agent()})
self.__session.verify = verify
if oauth2:
self.__session.headers.update({'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
def notify(cls, alert, *args, **kwargs):
current_span = extract_span_from_kwargs(**kwargs)
alert_def = alert['alert_def']
current_span.set_tag('alert_id', alert_def['id'])
entity = alert.get('entity')
is_changed = alert.get('alert_changed', False)
is_alert = alert.get('is_alert', False)
current_span.set_tag('entity', entity['id'])
current_span.set_tag('alert_changed', bool(is_changed))
current_span.set_tag('is_alert', is_alert)
url = kwargs.get('webhook',
cls._config.get('notifications.slack.webhook'))
repeat = kwargs.get('repeat', 0)
current_span.log_kv({'channel': kwargs.get('channel')})
if not url:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Missing webhook!'})
raise NotificationError('Webhook is required!')
message = {
'username': '******',
'channel': kwargs.get('channel', '#general'),
'text': kwargs.get('message', cls._get_subject(alert)),
'icon_emoji': ':bar_chart:',
}
headers = {
'User-agent': get_user_agent(),
'Content-type': 'application/json',
}
try:
logger.info('Sending to %s %s', url, message)
r = requests.post(url, json=message, headers=headers, timeout=5)
r.raise_for_status()
except Exception as e:
current_span.set_tag('error', True)
current_span.log_kv({'exception': str(e)})
logger.exception('Slack notification failed!')
return repeat
def test_elasticsearch_search_no_source_with_size(monkeypatch):
resp = resp_mock()
get = requests_mock(resp)
monkeypatch.setattr('requests.get', get)
url = 'http://es/'
es = ElasticsearchWrapper(url)
result = es.search(source=False, size=100)
assert result == resp.json.return_value
get.assert_called_with(get_full_url(url),
headers={'User-Agent': get_user_agent()},
params={'size': 100, '_source': 'false'},
timeout=10)
def test_elasticsearch_search(monkeypatch):
resp = resp_mock()
get = requests_mock(resp)
monkeypatch.setattr('requests.get', get)
url = 'http://es/'
es = ElasticsearchWrapper(url)
q = 'my-search'
result = es.search(q=q)
assert result == resp.json.return_value
get.assert_called_with(get_full_url(url),
headers={'User-Agent': get_user_agent()},
params={'q': q, 'size': DEFAULT_SIZE, '_source': 'true'},
timeout=10)
def test_elasticsearch_count(monkeypatch):
resp = resp_mock()
get = requests_mock(resp)
monkeypatch.setattr('requests.get', get)
url = 'http://es/'
es = ElasticsearchWrapper(url)
q = 'status:404'
result = es.count(q=q)
assert result == resp.json.return_value
get.assert_called_with(get_full_url(url, query_type=TYPE_COUNT),
headers={'User-Agent': get_user_agent()},
params={'q': q},
timeout=10)
def test_basicauth(monkeypatch):
resp = MagicMock()
resp.text = 'OK'
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://*****:*****@example.org', timeout=2)
assert 'OK' == http.text()
get.assert_called_with('http://example.org', auth=('user', 'pass'),
headers={'User-Agent': get_user_agent()},
params=None, timeout=2, verify=True, allow_redirects=True)
get.side_effect = requests.Timeout('timed out')
http = HttpWrapper('http://*****:*****@example.org')
with pytest.raises(HttpError) as ex:
http.text()
# verify that our basic auth credentials are not exposed in the exception message
assert 'HTTP request failed for http://example.org: timeout' == str(ex.value)
def test_basicauth(monkeypatch):
resp = MagicMock()
resp.text = 'OK'
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://*****:*****@example.org', timeout=2)
assert 'OK' == http.text()
get.assert_called_with('http://example.org', auth=('user', 'pass'),
headers={'User-Agent': get_user_agent()},
params=None, timeout=2, verify=True, allow_redirects=True)
get.side_effect = requests.Timeout('timed out')
http = HttpWrapper('http://*****:*****@example.org')
with pytest.raises(HttpError) as ex:
http.text()
# verify that our basic auth credentials are not exposed in the exception message
assert 'HTTP request failed for http://example.org: timeout' == str(ex.value)
def notify(cls, alert, per_entity=False, include_alert=True, message='', repeat=0, **kwargs):
url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'
repeat = kwargs.get('repeat', 0)
# Auth key!
service_key = kwargs.get('service_key', cls._config.get('notifications.pagerduty.servicekey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
if not service_key:
raise NotificationError('Service key is required!')
entity = alert.get('entity')
is_alert = alert.get('is_alert')
event_type = 'trigger' if is_alert else 'resolve'
alert_id = alert['alert_def']['id']
key = 'ZMON-{}'.format(alert_id) if not per_entity else 'ZMON-{}-{}'.format(alert_id, entity['id'])
description = message if message else cls._get_subject(alert)
message = {
'service_key': service_key,
'event_type': event_type,
'incident_key': key,
'description': description,
'client': 'ZMON',
'client_url': urlparse.urljoin(zmon_host, '/#/alert-details/{}'.format(alert_id)) if zmon_host else '',
'details': json.dumps(alert, cls=JsonDataEncoder) if include_alert else '',
}
try:
logger.info('Sending to %s %s', url, message)
headers = {'User-Agent': get_user_agent(), 'Content-type': 'application/json'}
r = requests.post(url, json=message, headers=headers, timeout=5)
r.raise_for_status()
except Exception as ex:
logger.exception('Notifying Pagerduty failed %s', ex)
return repeat
def test_elasticsearch_search_body(monkeypatch):
resp = resp_mock()
post = requests_mock(resp)
monkeypatch.setattr('requests.post', post)
url = 'http://es/'
es = ElasticsearchWrapper(url)
body = {'query': {'query_string': {'query': ''}}}
result = es.search(body=body)
assert result == resp.json.return_value
body['size'] = DEFAULT_SIZE
post.assert_called_with(get_full_url(url),
params={},
json=body,
headers={'User-Agent': get_user_agent()},
timeout=10)
def test_elasticsearch_health(monkeypatch):
resp = resp_mock()
get = requests_mock(resp)
monkeypatch.setattr('requests.get', get)
# mock tokens
token = 'TOKEN-123'
monkeypatch.setattr('tokens.get', lambda x: token)
url = 'http://es/'
es = ElasticsearchWrapper(url, oauth2=True)
result = es.health()
assert result == resp.json.return_value
get.assert_called_with(get_full_url(url, health=True),
headers={'User-Agent': get_user_agent(), 'Authorization': 'Bearer {}'.format(token)},
params=None,
timeout=10)
def __init__(self,
service_url,
infrastructure_account,
verify=True,
oauth2=False):
if not service_url:
raise ConfigurationError(
'EntitiesWrapper improperly configured. URL is missing!')
self.infrastructure_account = infrastructure_account
self.__service_url = urlparse.urljoin(service_url, 'api/v1/')
self.__session = requests.Session()
self.__session.headers.update({'User-Agent': get_user_agent()})
self.__session.verify = verify
if oauth2:
self.__session.headers.update(
{'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
def test_oauth2(monkeypatch):
resp = MagicMock()
resp.status_code = 218
resp.text = 'OK'
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
monkeypatch.setattr('tokens.get', lambda x: 'mytok' if x is 'uid' else 'myothertok')
http = HttpWrapper('http://example.org', oauth2=True, timeout=2)
assert 218 == http.code()
assert 'OK' == http.text()
get.assert_called_with('http://example.org', auth=None,
headers={'Authorization': 'Bearer mytok', 'User-Agent': get_user_agent()},
params=None, timeout=2, verify=True, allow_redirects=True)
http = HttpWrapper('http://example.org', oauth2=True, oauth2_token_name='foo', timeout=2)
assert 218 == http.code()
assert 'OK' == http.text()
get.assert_called_with('http://example.org', auth=None,
headers={'Authorization': 'Bearer myothertok', 'User-Agent': get_user_agent()},
params=None, timeout=2, verify=True, allow_redirects=True)
def __init__(self, url=None, username=None, password=None, es_url=None, index_prefix=''):
if not url:
raise RuntimeError('AppDynamics plugin improperly configured. URL is required!')
self.url = url
self.es_url = es_url
self.index_prefix = index_prefix
self.__oauth2 = False
self.__session = requests.Session()
if not username or not password:
self.__oauth2 = True
self.__session.headers.update({'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
else:
self.__session.auth = (username, password)
self.__session.headers.update({'User-Agent': get_user_agent()})
self.__session.params = {'output': 'json'}
self.__session.timeout = 3
def test_elasticsearch_search_no_source_body_with_size(monkeypatch):
resp = resp_mock()
post = requests_mock(resp)
monkeypatch.setattr('requests.post', post)
url = 'http://es/'
es = ElasticsearchWrapper(url)
body = {'query': {'query_string': {'query': ''}}}
indices = ['logstash-2016-*', 'logstash-2015-*']
result = es.search(indices=indices, body=body, source=False, size=100)
assert result == resp.json.return_value
body['size'] = 100
body['_source'] = False
post.assert_called_with(get_full_url(url, indices=indices),
params={},
json=body,
headers={'User-Agent': get_user_agent()},
timeout=10)
def notify(cls,
alert,
url=None,
body=None,
params=None,
headers=None,
timeout=5,
oauth2=False,
include_alert=True,
repeat=0):
urls = cls._config.get('notifications.http.whitelist.urls', [])
allow_any = cls._config.get('notifications.http.allow.all', False)
default_url = cls._config.get('notifications.http.default.url', None)
if isinstance(urls, basestring):
urls = urls.replace(' ', '').split(',')
if not url and not default_url:
raise NotificationError('URL is required!')
if not url:
url = default_url
elif not allow_any and url not in urls:
raise NotificationError(
'URL "{}" is not allowed. Please check worker white list URLs.'
.format(url))
if not is_absolute_http_url(url):
raise NotificationError('Absolute URL is required!')
# HTTP headers.
if not headers:
headers = {}
default_headers = cls._config.get('notifications.http.headers', {})
default_headers.update(headers)
if oauth2:
headers.update(
{'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
headers['User-Agent'] = get_user_agent()
if include_alert:
data = {
'alert': alert,
'body': body,
}
else:
data = body
try:
logger.info('Sending HTTP POST request to {}'.format(url))
r = requests.post(url,
data=json.dumps(data, cls=JsonDataEncoder),
params=params,
headers=headers,
timeout=timeout)
r.raise_for_status()
except:
logger.exception('Request failed!')
return repeat
def assert_client(cli):
# hack to access __session obj.
assert (USER, PASS) == cli._AppdynamicsWrapper__session.auth
assert get_user_agent() == cli._AppdynamicsWrapper__session.headers['User-Agent']
assert 'json' == cli._AppdynamicsWrapper__session.params['output']
import pytest
from mock import MagicMock
from zmon_worker_monitor.zmon_worker.common.http import get_user_agent
from zmon_worker_monitor.zmon_worker.notifications.slack import NotifySlack, NotificationError
URL = 'http://slack-webhook'
HEADERS = {
'User-agent': get_user_agent(),
'Content-type': 'application/json',
}
def test_slack_notification(monkeypatch):
post = MagicMock()
monkeypatch.setattr('requests.post', post)
alert = {'changed': True, 'is_alert': True, 'alert_def': {'id': 123, 'name': 'alert'}, 'entity': {'id': 'e-1'}}
NotifySlack._config = {'notifications.slack.webhook': URL}
r = NotifySlack.notify(alert, message='ALERT')
data = {
'username': '******',
'channel': '#general',
'text': 'ALERT',
'icon_emoji': ':bar_chart:',
}
def __init__(self, url, timeout=10, oauth2=False):
self.url = url
self.timeout = timeout
self.oauth2 = oauth2
self._headers = {'User-Agent': get_user_agent()}
def notify(cls,
alert,
url=None,
body=None,
params=None,
headers=None,
timeout=5,
oauth2=False,
include_alert=True,
repeat=0,
**kwargs):
current_span = extract_span_from_kwargs(**kwargs)
urls = cls._config.get('notifications.http.whitelist.urls', [])
allow_any = cls._config.get('notifications.http.allow.all', False)
default_url = cls._config.get('notifications.http.default.url', None)
alert_def = alert['alert_def']
current_span.set_tag('alert_id', alert_def['id'])
entity = alert.get('entity', {})
is_changed = alert.get('alert_changed', False)
is_alert = alert.get('is_alert', False)
current_span.set_tag('entity', entity.get('id'))
current_span.set_tag('alert_changed', bool(is_changed))
current_span.set_tag('is_alert', is_alert)
if isinstance(urls, basestring):
urls = urls.replace(' ', '').split(',')
if not url and not default_url:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Missing URL!'})
raise NotificationError('URL is required!')
if not url:
url = default_url
elif not allow_any and url not in urls:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'URL is not in whitelist'})
raise NotificationError(
'URL "{}" is not allowed. Please check worker white list URLs.'
.format(url))
if not is_absolute_http_url(url):
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Absolute URL required!'})
raise NotificationError('Absolute URL is required!')
# HTTP headers.
if not headers:
headers = {}
default_headers = cls._config.get('notifications.http.headers', {})
default_headers.update(headers)
if oauth2:
headers.update(
{'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
headers['User-Agent'] = get_user_agent()
if include_alert:
data = {
'alert': alert,
'body': body,
}
else:
data = body
try:
logger.info('Sending HTTP POST request to {}'.format(url))
r = requests.post(url,
data=json.dumps(data, cls=JsonDataEncoder),
params=params,
headers=headers,
timeout=timeout)
r.raise_for_status()
except Exception:
current_span.set_tag('error', True)
logger.exception('Request failed!')
return repeat
def notify(cls,
alert,
per_entity=False,
include_alert=True,
message='',
**kwargs):
url = 'https://api.opsgenie.com/v1/json/alert'
repeat = kwargs.get('repeat', 0)
# Auth key!
api_key = kwargs.get('api_key',
cls._config.get('notifications.opsgenie.apikey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
if not api_key:
raise NotificationError('API key is required!')
entity = alert.get('entity')
is_alert = alert.get('is_alert')
alert_id = alert['alert_def']['id']
alias = 'ZMON-{}'.format(
alert_id) if not per_entity else 'ZMON-{}-{}'.format(
alert_id, entity['id'])
note = urlparse.urljoin(
zmon_host,
'/#/alert-details/{}'.format(alert_id)) if zmon_host else ''
if is_alert:
data = {
'apiKey': api_key,
'alias': alias,
'message': message if message else cls._get_subject(alert),
'source': 'ZMON',
'entity': entity['id'],
'note': note,
'details': alert if include_alert else {},
}
else:
logger.info('Closing Opsgenie alert {}'.format(alias))
url = 'https://api.opsgenie.com/v1/json/alert/close'
data = {
'apiKey': api_key,
'alias': alias,
'source': 'ZMON',
'note': note,
}
try:
logger.info('Sending to %s %s', url, message)
headers = {
'User-Agent': get_user_agent(),
'Content-type': 'application/json'
}
r = requests.post(url,
data=json.dumps(data,
cls=JsonDataEncoder,
sort_keys=True),
headers=headers,
timeout=5)
r.raise_for_status()
except requests.HTTPError as e:
logger.error('HTTP Error ({}) {}'.format(e.response.status_code,
e.response.text))
except:
logger.exception('Notifying Opsgenie failed')
return repeat
def test_http(monkeypatch):
resp = MagicMock()
resp.status_code = 200
resp.text = '"foo"'
resp.content = resp.text
resp.json.return_value = 'foo'
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://example.org')
assert 200 == http.code()
assert '"foo"' == http.text()
assert 'foo' == http.json()
assert 5 == http.content_size()
get.assert_called_once_with('http://example.org', auth=None, headers={'User-Agent': get_user_agent()},
params=None, timeout=10, verify=True, allow_redirects=True)
resp.json.side_effect = Exception('JSON fail')
with pytest.raises(HttpError) as ex:
http.json()
assert 'JSON fail' == ex.value.message
def test_http_redirects(monkeypatch, fx_redirects):
kwargs, code = fx_redirects
exp_allow_redirects = False if 'allow_redirects' not in kwargs else kwargs['allow_redirects']
resp = MagicMock()
resp.status_code = code
resp.text = ''
redirect_url = 'http://example.org/some-file'
resp.headers = {'Location': redirect_url}
method = MagicMock()
method.return_value = resp
patch = 'requests.{}'.format(kwargs['method'].lower())
monkeypatch.setattr(patch, method)
http = HttpWrapper('http://example.org', **kwargs)
assert code == http.code()
assert '' == http.text()
assert redirect_url == http.headers()['Location']
method.assert_called_once_with('http://example.org', auth=None, headers={'User-Agent': get_user_agent()},
params=None, timeout=10, verify=True, allow_redirects=exp_allow_redirects)
def __client(self):
config = pykube.KubeConfig.from_service_account()
client = pykube.HTTPClient(config)
client.session.headers['User-Agent'] = "{} (check {})".format(get_user_agent(), self.__check_id)
client.session.trust_env = False
return client
def notify(cls,
alert,
teams=None,
per_entity=False,
include_alert=True,
message='',
**kwargs):
url = 'https://api.opsgenie.com/v2/alerts'
repeat = kwargs.get('repeat', 0)
# Auth key!
api_key = kwargs.get('api_key',
cls._config.get('notifications.opsgenie.apikey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
if not api_key:
raise NotificationError('API key is required!')
if not isinstance(teams, (list, basestring)):
raise NotificationError(
'Missing "teams" parameter. Either a team name or list of team names is required.'
)
if teams and isinstance(teams, basestring):
teams = [{'name': teams}]
else:
teams = [{'name': t} for t in teams]
entity = alert.get('entity')
is_changed = alert.get('alert_changed')
is_alert = alert.get('is_alert')
if not is_changed and not per_entity:
return repeat
alert_id = alert['alert_def']['id']
alias = 'ZMON-{}'.format(
alert_id) if not per_entity else 'ZMON-{}-{}'.format(
alert_id, entity['id'])
note = urlparse.urljoin(
zmon_host,
'/#/alert-details/{}'.format(alert_id)) if zmon_host else ''
details = {
'worker':
alert['worker'],
'id':
alert_id,
'name':
alert['alert_def']['name'],
'team':
alert['alert_def']['team'],
'responsible_team':
alert['alert_def']['responsible_team'],
'entity':
entity['id'],
'infrastructure_account':
entity.get('infrastructure_account', 'UNKNOWN'),
}
params = {}
if is_alert:
data = {
'alias': alias,
'teams': teams,
'message': message if message else cls._get_subject(alert),
'source': alert.get('worker', ''),
'entity': entity['id'],
'note': note,
'priority':
'P1' if int(alert['alert_def']['priority']) == 1 else 'P3',
'tags': alert['alert_def'].get('tags', [])
}
if include_alert:
data['details'] = details
else:
logger.info('Closing Opsgenie alert {}'.format(alias))
url = 'https://api.opsgenie.com/v2/alerts/{}/close'.format(alias)
data = {
'user': '******',
'source': alert.get('worker', 'ZMON Worker'),
'note': note,
}
params = {'identifierType': 'alias'}
try:
logger.info('Notifying Opsgenie %s %s', url, message)
headers = {
'User-Agent': get_user_agent(),
'Content-type': 'application/json',
'Authorization': 'GenieKey {}'.format(api_key),
}
r = requests.post(url,
data=json.dumps(data,
cls=JsonDataEncoder,
sort_keys=True),
headers=headers,
timeout=5,
params=params)
r.raise_for_status()
except requests.HTTPError as e:
logger.error('HTTP Error ({}) {}'.format(e.response.status_code,
e.response.text))
except:
logger.exception('Notifying Opsgenie failed')
return repeat
def notify(cls,
alert,
per_entity=False,
include_alert=True,
message='',
repeat=0,
**kwargs):
url = 'https://events.pagerduty.com/v2/enqueue'
repeat = kwargs.get('repeat', 0)
# Auth key!
routing_key = kwargs.get(
'routing_key',
cls._config.get('notifications.pagerduty.servicekey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
if not routing_key:
raise NotificationError('Service key is required!')
entity = alert.get('entity')
is_changed = alert.get('alert_changed')
is_alert = alert.get('is_alert')
if not is_changed and not per_entity:
return repeat
event_action = 'trigger' if is_alert else 'resolve'
alert_id = alert['alert_def']['id']
key = 'ZMON-{}'.format(
alert_id) if not per_entity else 'ZMON-{}-{}'.format(
alert_id, entity['id'])
description = message if message else cls._get_subject(
alert, include_event=False)
alert_class = kwargs.get('alert_class', '')
alert_group = kwargs.get('alert_group', '')
message = {
'routing_key':
routing_key,
'event_action':
event_action,
'dedup_key':
key,
'client':
'ZMON',
'client_url':
urlparse.urljoin(zmon_host, '/#/alert-details/{}'.format(alert_id))
if zmon_host else '',
'payload': {
'summary':
description,
'source':
alert.get('worker', ''),
'severity':
'critical'
if int(alert['alert_def']['priority']) == 1 else 'error',
'component':
entity['id'],
'custom_details':
alert if include_alert else {},
'class':
alert_class,
'group':
alert_group,
},
}
try:
logger.info('Notifying Pagerduty %s %s', url, message)
headers = {
'User-Agent': get_user_agent(),
'Content-type': 'application/json'
}
r = requests.post(url,
data=json.dumps(message, cls=JsonDataEncoder),
headers=headers,
timeout=5)
r.raise_for_status()
except:
logger.exception('Notifying Pagerduty failed')
return repeat
def notify(cls, alert, url=None, body=None, params=None, headers=None, timeout=5, oauth2=False, include_alert=True,
repeat=0, **kwargs):
current_span = extract_span_from_kwargs(**kwargs)
urls = cls._config.get('notifications.http.whitelist.urls', [])
allow_any = cls._config.get('notifications.http.allow.all', False)
default_url = cls._config.get('notifications.http.default.url', None)
alert_def = alert['alert_def']
current_span.set_tag('alert_id', alert_def['id'])
entity = alert.get('entity', {})
is_changed = alert.get('alert_changed', False)
is_alert = alert.get('is_alert', False)
current_span.set_tag('entity', entity.get('id'))
current_span.set_tag('alert_changed', bool(is_changed))
current_span.set_tag('is_alert', is_alert)
if isinstance(urls, basestring):
urls = urls.replace(' ', '').split(',')
if not url and not default_url:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Missing URL!'})
raise NotificationError('URL is required!')
if not url:
url = default_url
elif not allow_any and url not in urls:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'URL is not in whitelist'})
raise NotificationError('URL "{}" is not allowed. Please check worker white list URLs.'.format(url))
if not is_absolute_http_url(url):
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Absolute URL required!'})
raise NotificationError('Absolute URL is required!')
# HTTP headers.
if not headers:
headers = {}
default_headers = cls._config.get('notifications.http.headers', {})
default_headers.update(headers)
if oauth2:
headers.update({'Authorization': 'Bearer {}'.format(tokens.get('uid'))})
headers['User-Agent'] = get_user_agent()
if include_alert:
data = {
'alert': alert,
'body': body,
}
else:
data = body
try:
logger.info('Sending HTTP POST request to {}'.format(url))
r = requests.post(url, data=json.dumps(data, cls=JsonDataEncoder), params=params,
headers=headers, timeout=timeout)
r.raise_for_status()
except Exception:
current_span.set_tag('error', True)
logger.exception('Request failed!')
return repeat
def notify(cls,
alert,
teams=None,
per_entity=False,
include_alert=True,
include_captures=False,
priority=None,
message='',
description='',
custom_fields=None,
**kwargs):
current_span = extract_span_from_kwargs(**kwargs)
url = 'https://api.opsgenie.com/v2/alerts'
repeat = kwargs.get('repeat', 0)
# Auth key!
api_key = kwargs.get('api_key', cls._config.get('notifications.opsgenie.apikey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
entity = alert.get('entity')
is_changed = alert.get('alert_changed', False)
is_alert = alert.get('is_alert', False)
current_span.set_tag('entity', entity['id'])
current_span.set_tag('alert_changed', bool(is_changed))
current_span.set_tag('is_alert', is_alert)
alert_def = alert['alert_def']
current_span.set_tag('alert_id', alert_def['id'])
if not api_key:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'API key is required!'})
raise NotificationError('API key is required!')
if not isinstance(teams, (list, basestring)):
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Missing team!'})
raise NotificationError('Missing "teams" parameter. Either a team name or list of team names is required.')
current_span.log_kv({'teams': teams})
if priority and priority not in PRIORITIES:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Invalid priorities'})
raise NotificationError('Invalid priority. Valid values are: {}'.format(PRIORITIES))
if teams and isinstance(teams, basestring):
teams = [{'name': teams}]
else:
teams = [{'name': t} for t in teams]
if not is_changed and not per_entity:
return repeat
alert_id = alert['alert_def']['id']
alias = 'ZMON-{}'.format(alert_id) if not per_entity else 'ZMON-{}-{}'.format(alert_id, entity['id'])
note = alert_url = urlparse.urljoin(zmon_host, '/#/alert-details/{}'.format(alert_id)) if zmon_host else ''
if not priority:
priority = 'P1' if int(alert['alert_def']['priority']) == 1 else 'P3'
responsible_team = alert['alert_def'].get('responsible_team', teams[0]['name'])
msg = message if message else cls._get_subject(alert, include_event=False)
details = {
'alert_evaluation_ts': alert.get('alert_evaluation_ts', time.time())
}
alert_details = {
'worker': alert['worker'],
'zmon_team': alert['alert_def']['team'],
'entity': entity['id'],
'infrastructure_account': entity.get('infrastructure_account', 'UNKNOWN'),
'alert_url': alert_url,
}
params = {}
if is_alert:
tags = alert['alert_def'].get('tags', [])
tags.append(alert['alert_def']['id'])
data = {
'alias': alias,
'teams': teams,
'message': '[{}] - {}'.format(responsible_team, msg), # TODO: remove when it is no longer needed!
'source': alert.get('worker', ''),
'description': description,
'entity': entity['id'],
'note': note,
'priority': priority,
'tags': tags,
'details': details,
}
if isinstance(custom_fields, dict):
data['details'].update(custom_fields)
if include_alert:
data['details'].update(alert_details)
if include_captures:
data['details'].update(alert.get('captures'))
else:
logger.info('Closing Opsgenie alert {}'.format(alias))
url = 'https://api.opsgenie.com/v2/alerts/{}/close'.format(alias)
data = {
'user': '******',
'source': alert.get('worker', 'ZMON Worker'),
'note': note,
}
params = {'identifierType': 'alias'}
try:
logger.info('Notifying Opsgenie %s %s', url, message)
headers = {
'User-Agent': get_user_agent(),
'Content-type': 'application/json',
'Authorization': 'GenieKey {}'.format(api_key),
}
r = requests.post(url, data=json.dumps(data, cls=JsonDataEncoder, sort_keys=True), headers=headers,
timeout=5, params=params)
r.raise_for_status()
except requests.HTTPError as e:
current_span.set_tag('error', True)
logger.error('HTTP Error ({}) {}'.format(e.response.status_code, e.response.text))
except Exception:
current_span.set_tag('error', True)
current_span.log_kv({'exception': traceback.format_exc()})
logger.exception('Notifying Opsgenie failed')
return repeat
def notify(cls,
alert,
teams=None,
per_entity=False,
include_alert=True,
priority=None,
message='',
description='',
custom_fields=None,
**kwargs):
current_span = extract_span_from_kwargs(**kwargs)
url = 'https://api.opsgenie.com/v2/alerts'
repeat = kwargs.get('repeat', 0)
# Auth key!
api_key = kwargs.get('api_key',
cls._config.get('notifications.opsgenie.apikey'))
zmon_host = kwargs.get('zmon_host', cls._config.get('zmon.host'))
entity = alert.get('entity')
is_changed = alert.get('alert_changed', False)
is_alert = alert.get('is_alert', False)
current_span.set_tag('entity', entity['id'])
current_span.set_tag('alert_changed', bool(is_changed))
current_span.set_tag('is_alert', is_alert)
alert_def = alert['alert_def']
current_span.set_tag('alert_id', alert_def['id'])
if not api_key:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'API key is required!'})
raise NotificationError('API key is required!')
if not isinstance(teams, (list, basestring)):
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Missing team!'})
raise NotificationError(
'Missing "teams" parameter. Either a team name or list of team names is required.'
)
current_span.log_kv({'teams': teams})
if priority and priority not in PRIORITIES:
current_span.set_tag('notification_invalid', True)
current_span.log_kv({'reason': 'Invalid priorities'})
raise NotificationError(
'Invalid priority. Valid values are: {}'.format(PRIORITIES))
if teams and isinstance(teams, basestring):
teams = [{'name': teams}]
else:
teams = [{'name': t} for t in teams]
if not is_changed and not per_entity:
return repeat
alert_id = alert['alert_def']['id']
alias = 'ZMON-{}'.format(
alert_id) if not per_entity else 'ZMON-{}-{}'.format(
alert_id, entity['id'])
note = alert_url = urlparse.urljoin(
zmon_host,
'/#/alert-details/{}'.format(alert_id)) if zmon_host else ''
if not priority:
priority = 'P1' if int(
alert['alert_def']['priority']) == 1 else 'P3'
responsible_team = alert['alert_def'].get('responsible_team',
teams[0]['name'])
msg = message if message else cls._get_subject(alert,
include_event=False)
details = {
'alert_evaluation_ts': alert.get('alert_evaluation_ts',
time.time())
}
alert_details = {
'worker':
alert['worker'],
'zmon_team':
alert['alert_def']['team'],
'entity':
entity['id'],
'infrastructure_account':
entity.get('infrastructure_account', 'UNKNOWN'),
'alert_url':
alert_url,
}
params = {}
if is_alert:
tags = alert['alert_def'].get('tags', [])
tags.append(alert['alert_def']['id'])
data = {
'alias': alias,
'teams': teams,
'message': '[{}] - {}'.format(
responsible_team,
msg), # TODO: remove when it is no longer needed!
'source': alert.get('worker', ''),
'description': description,
'entity': entity['id'],
'note': note,
'priority': priority,
'tags': tags,
'details': details,
}
if isinstance(custom_fields, dict):
data['details'].update(custom_fields)
if include_alert:
data['details'].update(alert_details)
else:
logger.info('Closing Opsgenie alert {}'.format(alias))
url = 'https://api.opsgenie.com/v2/alerts/{}/close'.format(alias)
data = {
'user': '******',
'source': alert.get('worker', 'ZMON Worker'),
'note': note,
}
params = {'identifierType': 'alias'}
try:
logger.info('Notifying Opsgenie %s %s', url, message)
headers = {
'User-Agent': get_user_agent(),
'Content-type': 'application/json',
'Authorization': 'GenieKey {}'.format(api_key),
}
r = requests.post(url,
data=json.dumps(data,
cls=JsonDataEncoder,
sort_keys=True),
headers=headers,
timeout=5,
params=params)
r.raise_for_status()
except requests.HTTPError as e:
current_span.set_tag('error', True)
logger.error('HTTP Error ({}) {}'.format(e.response.status_code,
e.response.text))
except Exception:
current_span.set_tag('error', True)
current_span.log_kv({'exception': traceback.format_exc()})
logger.exception('Notifying Opsgenie failed')
return repeat
import pytest
from mock import MagicMock
from zmon_worker_monitor.zmon_worker.common.http import get_user_agent
from zmon_worker_monitor.zmon_worker.notifications.slack import NotifySlack, NotificationError
URL = 'http://slack-webhook'
HEADERS = {
'User-agent': get_user_agent(),
'Content-type': 'application/json',
}
def test_slack_notification(monkeypatch):
post = MagicMock()
monkeypatch.setattr('requests.post', post)
alert = {
'changed': True,
'is_alert': True,
'alert_def': {
'id': 123,
'name': 'alert'
},
'entity': {
'id': 'e-1'
}
}
NotifySlack._config = {'notifications.slack.webhook': URL}
