def test_retries(monkeypatch):
session = MagicMock()
session.return_value.get.return_value.text = 'OK'
monkeypatch.setattr('requests.Session', session)
http = HttpWrapper('http://example.org', max_retries=10)
assert 'OK' == http.text()
assert session.return_value.get.called
def test_retries(monkeypatch):
session = MagicMock()
session.return_value.get.return_value.text = "OK"
monkeypatch.setattr("requests.Session", session)
http = HttpWrapper("http://example.org", max_retries=10)
assert "OK" == http.text()
assert session.return_value.get.called
def test_http_actuator_metrics_valid(monkeypatch, metrics_response, expected):
resp = MagicMock()
resp.json.return_value = metrics_response
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://example.org')
assert expected == http.actuator_metrics()
def test_http_actuator_metrics_invalid(monkeypatch):
resp = MagicMock()
resp.json.return_value = 'foo'
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://example.org')
with pytest.raises(HttpError) as ex:
http.actuator_metrics()
assert 'Invalid actuator metrics: response must be a JSON object' == ex.value.message
def test_http_prometheus_flat_with_filtering(monkeypatch):
resp = MagicMock()
resp.text = '''
# HELP http_server_requests_seconds
# TYPE http_server_requests_seconds summary
http_server_requests_seconds{exception="None",method="GET",status="200",uri="/api/hello",quantile="0.95",} 0.003080192
http_server_requests_seconds{exception="None",method="GET",status="200",uri="/api/hello",quantile="0.99",} 0.071237632
http_server_requests_seconds_count{exception="None",method="GET",status="200",uri="/api/hello",} 20.0
http_server_requests_seconds_sum{exception="None",method="GET",status="200",uri="/api/hello",} 0.103182669
# HELP http_server_requests_seconds_max
# TYPE http_server_requests_seconds_max gauge
http_server_requests_seconds_max{exception="None",method="GET",status="200",uri="/api/hello",} 0.067652582
# HELP jvm_memory_committed_bytes The amount of memory in bytes that is committed for the Java virtual machine to use
# TYPE jvm_memory_committed_bytes gauge
jvm_memory_committed_bytes{area="nonheap",id="Code Cache",} 1.9070976E7
jvm_memory_committed_bytes{area="nonheap",id="Metaspace",} 5.5574528E7
jvm_memory_committed_bytes{area="nonheap",id="Compressed Class Space",} 7340032.0
jvm_memory_committed_bytes{area="heap",id="PS Eden Space",} 2.84688384E8
jvm_memory_committed_bytes{area="heap",id="PS Survivor Space",} 1.6252928E7
jvm_memory_committed_bytes{area="heap",id="PS Old Gen",} 2.3855104E8
# HELP httpsessions_max httpsessions_max
# TYPE httpsessions_max gauge
httpsessions_max -1.0
# HELP httpsessions_active httpsessions_active
# TYPE httpsessions_active gauge
httpsessions_active 0.0
# HELP mem mem
# TYPE mem gauge
mem 370583.0
# HELP mem_free mem_free
# TYPE mem_free gauge
mem_free 176263.0
# HELP processors processors
# TYPE processors gauge
processors 8.0
'''
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://example.org/prometheus/')
expected = {
u'httpsessions_max': -1.0,
u'http_server_requests_seconds_sum': {
u'exception.None.method.GET.status.200.uri./api/hello': 0.103182669
}
}
assert expected == http.prometheus_flat([
u'httpsessions_max',
u'http_server_requests_seconds_sum.exception.None.method.GET.status.200.uri./api/hello'
])
def test_http_prometheus(monkeypatch):
resp = MagicMock()
# see http://prometheus.io/docs/instrumenting/exposition_formats/#text-format-details
resp.text = '''
# HELP api_http_request_count The total number of HTTP requests.
# TYPE api_http_request_count counter
http_request_count{method="post",code="200"} 1027 1395066363000
http_request_count{method="post",code="400"} 3 1395066363000
'''
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://example.org/prometheus/')
expected = {u'http_request_count': [({u'code': u'200', u'method': u'post'}, 1027.0),
({u'code': u'400', u'method': u'post'}, 3.0)]}
assert expected == http.prometheus()
def test_basicauth(monkeypatch):
resp = MagicMock()
resp.text = 'OK'
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://*****:*****@example.org', timeout=2)
assert 'OK' == http.text()
get.assert_called_with('http://example.org', auth=('user', 'pass'),
headers={'User-Agent': get_user_agent()},
params=None, timeout=2, verify=True, allow_redirects=True)
get.side_effect = requests.Timeout('timed out')
http = HttpWrapper('http://*****:*****@example.org')
with pytest.raises(HttpError) as ex:
http.text()
# verify that our basic auth credentials are not exposed in the exception message
assert 'HTTP request failed for http://example.org: timeout' == str(ex.value)
def test_oauth2(monkeypatch):
resp = MagicMock()
resp.status_code = 218
resp.text = "OK"
get = MagicMock()
get.return_value = resp
monkeypatch.setattr("requests.get", get)
monkeypatch.setattr("tokens.get", lambda x: "mytok")
http = HttpWrapper("http://example.org", oauth2=True, timeout=2)
assert 218 == http.code()
assert "OK" == http.text()
get.assert_called_once_with(
"http://example.org",
auth=None,
headers={"Authorization": "Bearer mytok", "User-Agent": "zmon-worker/0.1"},
params=None,
timeout=2,
verify=True,
)
def test_http_prometheus(monkeypatch):
resp = MagicMock()
# see http://prometheus.io/docs/instrumenting/exposition_formats/#text-format-details
resp.text = """
# HELP api_http_request_count The total number of HTTP requests.
# TYPE api_http_request_count counter
http_request_count{method="post",code="200"} 1027 1395066363000
http_request_count{method="post",code="400"} 3 1395066363000
"""
get = MagicMock()
get.return_value = resp
monkeypatch.setattr("requests.get", get)
http = HttpWrapper("http://example.org/prometheus/")
expected = {
u"http_request_count": [
({u"code": u"200", u"method": u"post"}, 1027.0),
({u"code": u"400", u"method": u"post"}, 3.0),
]
}
assert expected == http.prometheus()
def test_http_certs(monkeypatch, url, port, err):
socket_mock = MagicMock()
sock = MagicMock()
ssl_sock = MagicMock()
wrap = MagicMock()
socket_mock.return_value = sock
ssl_sock.connect.return_value = True
ssl_sock.getpeercert.return_value = {
'issuer': '123',
'notAfter': 'Apr 23 17:57:00 2017 GMT',
'notBefore': 'Jan 23 17:57:00 2017 GMT',
}
wrap.return_value = ssl_sock
monkeypatch.setattr('socket.socket', socket_mock)
monkeypatch.setattr('ssl.wrap_socket', wrap)
http = HttpWrapper(url)
if err:
with pytest.raises(err):
res = http.certs()
else:
res = http.certs()
assert [{
'issuer': '123',
'notAfter': 'Apr 23 17:57:00 2017 GMT',
'notBefore': 'Jan 23 17:57:00 2017 GMT',
'not_after': '2017-04-23 17:57:00',
'not_before': '2017-01-23 17:57:00',
}] == res
sock.settimeout.assert_called_with(10)
ssl_sock.connect.assert_called_with(('zmon', port))
ssl_sock.close.assert_called()
def test_http_redirects(monkeypatch, fx_redirects):
kwargs, code = fx_redirects
exp_allow_redirects = False if 'allow_redirects' not in kwargs else kwargs['allow_redirects']
resp = MagicMock()
resp.status_code = code
resp.text = ''
redirect_url = 'http://example.org/some-file'
resp.headers = {'Location': redirect_url}
method = MagicMock()
method.return_value = resp
patch = 'requests.{}'.format(kwargs['method'].lower())
monkeypatch.setattr(patch, method)
http = HttpWrapper('http://example.org', **kwargs)
assert code == http.code()
assert '' == http.text()
assert redirect_url == http.headers()['Location']
method.assert_called_once_with('http://example.org', auth=None, headers={'User-Agent': get_user_agent()},
params=None, timeout=10, verify=True, allow_redirects=exp_allow_redirects)
def test_basicauth(monkeypatch):
resp = MagicMock()
resp.text = "OK"
get = MagicMock()
get.return_value = resp
monkeypatch.setattr("requests.get", get)
http = HttpWrapper("http://*****:*****@example.org", timeout=2)
assert "OK" == http.text()
get.assert_called_with(
"http://example.org",
auth=("user", "pass"),
headers={"User-Agent": "zmon-worker/0.1"},
params=None,
timeout=2,
verify=True,
)
get.side_effect = requests.Timeout("timed out")
http = HttpWrapper("http://*****:*****@example.org")
with pytest.raises(HttpError) as ex:
http.text()
# verify that our basic auth credentials are not exposed in the exception message
assert "HTTP request failed for http://example.org: timeout" == str(ex.value)
def test_http_jolokia(monkeypatch):
resp = MagicMock()
resp.json.return_value = {"foo": "bar"}
post = MagicMock()
post.return_value = resp
monkeypatch.setattr("requests.post", post)
with pytest.raises(HttpError) as ex:
http = HttpWrapper("http://example.org/foo")
http.jolokia([])
assert "URL needs to end in jolokia/ and not contain ? and &" == ex.value.message
http = HttpWrapper("http://example.org/jolokia/")
assert {"foo": "bar"} == http.jolokia(
[{"mbean": "java.lang:type=Memory", "attribute": "HeapMemoryUsage", "path": "used"}]
)
resp.json.side_effect = Exception("JSON FAIL")
http = HttpWrapper("http://example.org/jolokia/")
with pytest.raises(HttpError) as ex:
http.jolokia([{"mbean": "java.lang:type=Memory", "attribute": "HeapMemoryUsage", "path": "used"}])
assert "JSON FAIL" == ex.value.message
def test_http(monkeypatch):
resp = MagicMock()
resp.status_code = 200
resp.text = '"foo"'
resp.content = resp.text
resp.json.return_value = "foo"
get = MagicMock()
get.return_value = resp
monkeypatch.setattr("requests.get", get)
http = HttpWrapper("http://example.org")
assert 200 == http.code()
assert '"foo"' == http.text()
assert "foo" == http.json()
assert 5 == http.content_size()
resp.json.side_effect = Exception("JSON fail")
with pytest.raises(HttpError) as ex:
http.json()
assert "JSON fail" == ex.value.message
def test_http(monkeypatch):
resp = MagicMock()
resp.status_code = 200
resp.text = '"foo"'
resp.content = resp.text
resp.json.return_value = 'foo'
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://example.org')
assert 200 == http.code()
assert '"foo"' == http.text()
assert 'foo' == http.json()
assert 5 == http.content_size()
get.assert_called_once_with('http://example.org', auth=None, headers={'User-Agent': get_user_agent()},
params=None, timeout=10, verify=True, allow_redirects=True)
resp.json.side_effect = Exception('JSON fail')
with pytest.raises(HttpError) as ex:
http.json()
assert 'JSON fail' == ex.value.message
def test_oauth2(monkeypatch):
resp = MagicMock()
resp.status_code = 218
resp.text = 'OK'
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
monkeypatch.setattr('tokens.get', lambda x: 'mytok' if x is 'uid' else 'myothertok')
http = HttpWrapper('http://example.org', oauth2=True, timeout=2)
assert 218 == http.code()
assert 'OK' == http.text()
get.assert_called_with('http://example.org', auth=None,
headers={'Authorization': 'Bearer mytok', 'User-Agent': get_user_agent()},
params=None, timeout=2, verify=True, allow_redirects=True)
http = HttpWrapper('http://example.org', oauth2=True, oauth2_token_name='foo', timeout=2)
assert 218 == http.code()
assert 'OK' == http.text()
get.assert_called_with('http://example.org', auth=None,
headers={'Authorization': 'Bearer myothertok', 'User-Agent': get_user_agent()},
params=None, timeout=2, verify=True, allow_redirects=True)
def test_http_jolokia(monkeypatch):
resp = MagicMock()
resp.json.return_value = {'foo': 'bar'}
post = MagicMock()
post.return_value = resp
monkeypatch.setattr('requests.post', post)
with pytest.raises(HttpError) as ex:
http = HttpWrapper('http://example.org/foo')
http.jolokia([])
assert 'URL needs to end in /jolokia/ and not contain ? and &' == ex.value.message
with pytest.raises(HttpError) as ex:
http = HttpWrapper('http://example.org/foo#/jolokia/')
http.jolokia([])
assert 'URL needs to end in /jolokia/ and not contain ? and &' == ex.value.message
with pytest.raises(CheckError) as ex:
http = HttpWrapper('http://example.org/jolokia/')
http.jolokia([{'foo': 'bar'}])
assert 'missing "mbean" key in read request' == ex.value.message
http = HttpWrapper('http://example.org/jolokia/')
assert {'foo': 'bar'} == http.jolokia([{
"mbean": "java.lang:type=Memory",
"attribute": "HeapMemoryUsage",
"path": "used",
}])
resp.json.side_effect = Exception('JSON FAIL')
http = HttpWrapper('http://example.org/jolokia/')
with pytest.raises(HttpError) as ex:
http.jolokia([{
"mbean": "java.lang:type=Memory",
"attribute": "HeapMemoryUsage",
"path": "used",
}])
assert 'JSON FAIL' == ex.value.message
def test_http_invalid_method(method):
with pytest.raises(CheckError):
HttpWrapper('http://example.org', method=method)
def test_http_invalid_base_url():
with pytest.raises(ConfigurationError):
HttpWrapper(':9000', base_url=None)
def test_http_jolokia(monkeypatch):
resp = MagicMock()
resp.json.return_value = {'foo': 'bar'}
post = MagicMock()
post.return_value = resp
monkeypatch.setattr('requests.post', post)
with pytest.raises(HttpError) as ex:
http = HttpWrapper('http://example.org/foo')
http.jolokia([])
assert 'URL needs to end in /jolokia/ and not contain ? and &' == ex.value.message
with pytest.raises(HttpError) as ex:
http = HttpWrapper('http://example.org/foo#/jolokia/')
http.jolokia([])
assert 'URL needs to end in /jolokia/ and not contain ? and &' == ex.value.message
with pytest.raises(CheckError) as ex:
http = HttpWrapper('http://example.org/jolokia/')
http.jolokia([{'foo': 'bar'}])
assert 'missing "mbean" key in read request' == ex.value.message
http = HttpWrapper('http://example.org/jolokia/')
assert {'foo': 'bar'} == http.jolokia([{
"mbean": "java.lang:type=Memory",
"attribute": "HeapMemoryUsage",
"path": "used",
}])
resp.json.side_effect = Exception('JSON FAIL')
http = HttpWrapper('http://example.org/jolokia/')
with pytest.raises(HttpError) as ex:
http.jolokia([{
"mbean": "java.lang:type=Memory",
"attribute": "HeapMemoryUsage",
"path": "used",
}])
assert 'JSON FAIL' == ex.value.message
def test_http_errors(monkeypatch):
resp = MagicMock()
resp.status_code = 404
resp.raise_for_status.side_effect = requests.HTTPError('Not Found')
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://example.org')
# the code method will not raise an exception..
assert 404 == http.code()
for meth in ('time', 'json', 'cookies', 'headers'):
with pytest.raises(HttpError) as ex:
# ..but other methods will!
getattr(http, meth)()
assert 'Not Found' == ex.value.message
get.side_effect = requests.Timeout('timed out')
http = HttpWrapper('http://example.org')
with pytest.raises(HttpError) as ex:
http.time()
assert 'timeout' == ex.value.message
get.side_effect = requests.ConnectionError('connfail')
http = HttpWrapper('http://example.org')
with pytest.raises(HttpError) as ex:
http.code()
assert 'connection failed' == ex.value.message
get.side_effect = Exception('foofail')
http = HttpWrapper('http://example.org')
with pytest.raises(HttpError) as ex:
http.code()
assert 'foofail' == ex.value.message
def test_http_errors(monkeypatch):
resp = MagicMock()
resp.status_code = 404
resp.raise_for_status.side_effect = requests.HTTPError("Not Found")
get = MagicMock()
get.return_value = resp
monkeypatch.setattr("requests.get", get)
http = HttpWrapper("http://example.org")
# the code method will not raise an exception..
assert 404 == http.code()
for meth in ("time", "json", "cookies", "headers"):
with pytest.raises(HttpError) as ex:
# ..but other methods will!
getattr(http, meth)()
assert "Not Found" == ex.value.message
get.side_effect = requests.Timeout("timed out")
http = HttpWrapper("http://example.org")
with pytest.raises(HttpError) as ex:
http.time()
assert "timeout" == ex.value.message
get.side_effect = requests.ConnectionError("connfail")
http = HttpWrapper("http://example.org")
with pytest.raises(HttpError) as ex:
http.code()
assert "connection failed" == ex.value.message
get.side_effect = Exception("foofail")
http = HttpWrapper("http://example.org")
with pytest.raises(HttpError) as ex:
http.code()
assert "foofail" == ex.value.message
def test_http_errors(monkeypatch):
resp = MagicMock()
resp.status_code = 404
resp.raise_for_status.side_effect = requests.HTTPError('Not Found')
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://example.org')
# the code method will not raise an exception..
assert 404 == http.code()
for meth in ('time', 'json', 'cookies', 'headers'):
with pytest.raises(HttpError) as ex:
# ..but other methods will!
getattr(http, meth)()
assert 'Not Found' == ex.value.message
get.side_effect = requests.Timeout('timed out')
http = HttpWrapper('http://example.org')
with pytest.raises(HttpError) as ex:
http.time()
assert 'timeout' == ex.value.message
get.side_effect = requests.ConnectionError('connfail')
http = HttpWrapper('http://example.org')
with pytest.raises(HttpError) as ex:
http.code()
assert 'connection failed' == ex.value.message
get.side_effect = Exception('foofail')
http = HttpWrapper('http://example.org')
with pytest.raises(HttpError) as ex:
http.code()
assert 'foofail' == ex.value.message
def test_http_prometheus_flat(monkeypatch):
resp = MagicMock()
resp.text = '''
# HELP http_server_requests_seconds
# TYPE http_server_requests_seconds summary
http_server_requests_seconds{exception="None",method="GET",status="200",uri="/api/hello",quantile="0.95",} 0.003080192
http_server_requests_seconds{exception="None",method="GET",status="200",uri="/api/hello",quantile="0.99",} 0.071237632
http_server_requests_seconds_count{exception="None",method="GET",status="200",uri="/api/hello",} 20.0
http_server_requests_seconds_sum{exception="None",method="GET",status="200",uri="/api/hello",} 0.103182669
# HELP http_server_requests_seconds_max
# TYPE http_server_requests_seconds_max gauge
http_server_requests_seconds_max{exception="None",method="GET",status="200",uri="/api/hello",} 0.067652582
# HELP jvm_memory_committed_bytes The amount of memory in bytes that is committed for the Java virtual machine to use
# TYPE jvm_memory_committed_bytes gauge
jvm_memory_committed_bytes{area="nonheap",id="Code Cache",} 1.9070976E7
jvm_memory_committed_bytes{area="nonheap",id="Metaspace",} 5.5574528E7
jvm_memory_committed_bytes{area="nonheap",id="Compressed Class Space",} 7340032.0
jvm_memory_committed_bytes{area="heap",id="PS Eden Space",} 2.84688384E8
jvm_memory_committed_bytes{area="heap",id="PS Survivor Space",} 1.6252928E7
jvm_memory_committed_bytes{area="heap",id="PS Old Gen",} 2.3855104E8
# HELP httpsessions_max httpsessions_max
# TYPE httpsessions_max gauge
httpsessions_max -1.0
# HELP httpsessions_active httpsessions_active
# TYPE httpsessions_active gauge
httpsessions_active 0.0
# HELP mem mem
# TYPE mem gauge
mem 370583.0
# HELP mem_free mem_free
# TYPE mem_free gauge
mem_free 176263.0
# HELP processors processors
# TYPE processors gauge
processors 8.0
'''
get = MagicMock()
get.return_value = resp
monkeypatch.setattr('requests.get', get)
http = HttpWrapper('http://example.org/prometheus/')
expected = {
u'jvm_memory_committed_bytes': {
u'area.nonheap.id.Code Cache': 1.9070976E7,
u'area.nonheap.id.Metaspace': 5.5574528E7,
u'area.nonheap.id.Compressed Class Space': 7340032.0,
u'area.heap.id.PS Eden Space': 2.84688384E8,
u'area.heap.id.PS Survivor Space': 1.6252928E7,
u'area.heap.id.PS Old Gen': 2.3855104E8
},
u'httpsessions_max': -1.0,
u'httpsessions_active': 0.0,
u'mem': 370583.0,
u'mem_free': 176263.0,
u'processors': 8.0,
u'http_server_requests_seconds': {
u'exception.None.method.GET.quantile.0.95.status.200.uri./api/hello': 0.003080192,
u'exception.None.method.GET.quantile.0.99.status.200.uri./api/hello': 0.071237632,
},
u'http_server_requests_seconds_count': {
u'exception.None.method.GET.status.200.uri./api/hello': 20.0
},
u'http_server_requests_seconds_max': {
u'exception.None.method.GET.status.200.uri./api/hello': 0.067652582
},
u'http_server_requests_seconds_sum': {
u'exception.None.method.GET.status.200.uri./api/hello': 0.103182669
}
}
assert expected == http.prometheus_flat()
