def msgpack_nodes(nodes):
nodes_info = []
for node in nodes:
info = dict(path=node.path, is_leaf=node.is_leaf)
if node.is_leaf:
info['intervals'] = [interval.tuple for interval in node.intervals]
nodes_info.append(info)
return msgpack.dumps(nodes_info)
def msgpack_nodes(nodes):
nodes_info = []
# make sure everything is unicode in python 2.x and 3.x
for node in nodes:
info = {
text_type('path'): text_type(node.path),
text_type('is_leaf'): node.is_leaf,
}
if node.is_leaf:
info[text_type('intervals')] = [interval.tuple for interval in node.intervals]
nodes_info.append(info)
return msgpack.dumps(nodes_info, use_bin_type=True)
def test_RemoteReader_fetch_multi(self, http_request):
test_finders = RemoteFinder.factory()
finder = test_finders[0]
startTime = 1496262000
endTime = 1496262060
# no path or bulk_query
reader = RemoteReader(finder,{})
self.assertEqual(reader.bulk_query, [])
result = reader.fetch_multi(startTime, endTime)
self.assertEqual(result, [])
self.assertEqual(http_request.call_count, 0)
# path
reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'})
data = [
{'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}
]
responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)), status=200, preload_content=False)
http_request.return_value = responseObject
result = reader.fetch_multi(startTime, endTime)
expected_response = [
{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
}
]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://127.0.0.1/render/',
))
self.assertEqual(http_request.call_args[1], {
'fields': [
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
],
'headers': None,
'preload_content': False,
'timeout': 10,
})
# bulk_query & now
finder = test_finders[1]
reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'}, bulk_query=['a.b.c.d'])
data = [
{'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}
]
responseObject = HTTPResponse(
body=BytesIO(msgpack.dumps(data, use_bin_type=True)),
status=200,
preload_content=False,
headers={'Content-Type': 'application/x-msgpack'}
)
http_request.return_value = responseObject
result = reader.fetch_multi(startTime, endTime, now=endTime, requestContext={'forwardHeaders': {'Authorization': 'Basic xxxx'}})
expected_response = [
{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
}
]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://8.8.8.8/graphite/render/',
))
self.assertEqual(http_request.call_args[1], {
'fields': [
('format', 'msgpack'),
('local', '0'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
('now', endTime),
],
'headers': {'Authorization': 'Basic xxxx'},
'preload_content': False,
'timeout': 10,
})
# non-pickle response
responseObject = HTTPResponse(body=BytesIO(b'error'), status=200, preload_content=False)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception, 'Error decoding response from http://[^ ]+: .+'):
reader.fetch(startTime, endTime)
# invalid response data
data = [
{},
]
responseObject = HTTPResponse(
body=BytesIO(msgpack.dumps(data, use_bin_type=True)),
status=200,
preload_content=False,
headers={'Content-Type': 'application/x-msgpack'}
)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception, 'Invalid render response from http://[^ ]+: KeyError\(\'name\',?\)'):
reader.fetch(startTime, endTime)
# non-200 response
responseObject = HTTPResponse(body=BytesIO(b'error'), status=500, preload_content=False)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception, 'Error response 500 from http://[^ ]+'):
reader.fetch(startTime, endTime)
# exception raised by request()
http_request.side_effect = Exception('error')
with self.assertRaisesRegexp(Exception, 'Error requesting http://[^ ]+: error'):
reader.fetch(startTime, endTime)
def test_RemoteReader_fetch_multi(self, http_request):
test_finders = RemoteFinder.factory()
finder = test_finders[0]
startTime = 1496262000
endTime = 1496262060
# no path or bulk_query
reader = RemoteReader(finder, {})
self.assertEqual(reader.bulk_query, [])
result = reader.fetch_multi(startTime, endTime)
self.assertEqual(result, [])
self.assertEqual(http_request.call_count, 0)
# path
reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'})
data = [{
'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}]
responseObject = HTTPResponse(body=StringIO(pickle.dumps(data)),
status=200,
preload_content=False)
http_request.return_value = responseObject
result = reader.fetch_multi(startTime, endTime)
expected_response = [{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
}]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://127.0.0.1/render/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
],
'headers':
None,
'preload_content':
False,
'timeout':
10,
})
# bulk_query & now
finder = test_finders[1]
reader = RemoteReader(finder, {
'intervals': [],
'path': 'a.b.c.d'
},
bulk_query=['a.b.c.d'])
data = [{
'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}]
responseObject = HTTPResponse(
body=StringIO(msgpack.dumps(data)),
status=200,
preload_content=False,
headers={'Content-Type': 'application/x-msgpack'})
http_request.return_value = responseObject
result = reader.fetch_multi(
startTime,
endTime,
now=endTime,
requestContext={'forwardHeaders': {
'Authorization': 'Basic xxxx'
}})
expected_response = [{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
}]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://8.8.8.8/graphite/render/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('format', 'msgpack'),
('local', '0'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
('now', endTime),
],
'headers': {
'Authorization': 'Basic xxxx'
},
'preload_content':
False,
'timeout':
10,
})
# non-pickle response
responseObject = HTTPResponse(body=StringIO('error'),
status=200,
preload_content=False)
http_request.return_value = responseObject
with self.assertRaisesRegexp(
Exception,
'Error decoding render response from http://[^ ]+: .+'):
reader.fetch(startTime, endTime)
# non-200 response
responseObject = HTTPResponse(body=StringIO('error'),
status=500,
preload_content=False)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception,
'Error response 500 from http://[^ ]+'):
reader.fetch(startTime, endTime)
# exception raised by request()
http_request.side_effect = Exception('error')
with self.assertRaisesRegexp(Exception,
'Error requesting http://[^ ]+: error'):
reader.fetch(startTime, endTime)
def test_find_nodes(self, http_request):
finder = RemoteFinder('127.0.0.1')
startTime = 1496262000
endTime = 1496262060
data = [
{
'path': 'a.b.c',
'is_leaf': False,
},
{
'path': 'a.b.c.d',
'is_leaf': True,
},
]
responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)),
status=200,
preload_content=False)
http_request.return_value = responseObject
query = FindQuery('a.b.c', startTime, endTime)
result = finder.find_nodes(query)
self.assertIsInstance(result, types.GeneratorType)
nodes = list(result)
self.assertEqual(http_request.call_args[0], (
'POST',
'http://127.0.0.1/metrics/find/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('local', '1'),
('format', 'pickle'),
('query', 'a.b.c'),
('from', startTime),
('until', endTime),
],
'headers':
None,
'preload_content':
False,
'timeout':
10,
})
self.assertEqual(len(nodes), 2)
self.assertIsInstance(nodes[0], BranchNode)
self.assertEqual(nodes[0].path, 'a.b.c')
self.assertIsInstance(nodes[1], LeafNode)
self.assertEqual(nodes[1].path, 'a.b.c.d')
finder = RemoteFinder('https://127.0.0.1?format=msgpack')
data = [
{
'path': 'a.b.c',
'is_leaf': False,
},
{
'path': 'a.b.c.d',
'is_leaf': True,
},
]
responseObject = HTTPResponse(
body=BytesIO(msgpack.dumps(data, use_bin_type=True)),
status=200,
preload_content=False,
headers={'Content-Type': 'application/x-msgpack'})
http_request.return_value = responseObject
query = FindQuery('a.b.c', None, None)
result = finder.find_nodes(query)
self.assertIsInstance(result, types.GeneratorType)
nodes = list(result)
self.assertEqual(http_request.call_args[0], (
'POST',
'https://127.0.0.1/metrics/find/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('local', '1'),
('format', 'msgpack'),
('query', 'a.b.c'),
],
'headers':
None,
'preload_content':
False,
'timeout':
10,
})
self.assertEqual(len(nodes), 2)
self.assertIsInstance(nodes[0], BranchNode)
self.assertEqual(nodes[0].path, 'a.b.c')
self.assertIsInstance(nodes[1], LeafNode)
self.assertEqual(nodes[1].path, 'a.b.c.d')
# non-pickle response
responseObject = HTTPResponse(body=BytesIO(b'error'),
status=200,
preload_content=False)
http_request.return_value = responseObject
result = finder.find_nodes(query)
with self.assertRaisesRegexp(
Exception,
'Error decoding find response from https://[^ ]+: .+'):
list(result)
def _test_find_nodes(self, http_request):
finder = RemoteFinder('127.0.0.1')
startTime = 1496262000
endTime = 1496262060
data = [
{
'path': 'a.b.c',
'is_leaf': False,
},
{
'path': 'a.b.c.d',
'is_leaf': True,
},
]
responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)), status=200, preload_content=False)
http_request.return_value = responseObject
query = FindQuery('a.b.c', startTime, endTime)
nodes = finder.find_nodes(query)
self.assertEqual(http_request.call_args[0], (
'POST',
'http://127.0.0.1/metrics/find/',
))
self.assertEqual(http_request.call_args[1], {
'fields': [
('local', '1'),
('format', 'pickle'),
('query', 'a.b.c'),
('from', startTime),
('until', endTime),
],
'headers': None,
'preload_content': False,
'timeout': 10,
})
self.assertEqual(len(nodes), 2)
self.assertIsInstance(nodes[0], BranchNode)
self.assertEqual(nodes[0].path, 'a.b.c')
self.assertIsInstance(nodes[1], LeafNode)
self.assertEqual(nodes[1].path, 'a.b.c.d')
finder = RemoteFinder('https://127.0.0.1?format=msgpack')
data = [
{
'path': 'a.b.c',
'is_leaf': False,
},
{
'path': 'a.b.c.d',
'is_leaf': True,
},
]
responseObject = HTTPResponse(
body=BytesIO(msgpack.dumps(data, use_bin_type=True)),
status=200,
preload_content=False,
headers={'Content-Type': 'application/x-msgpack'}
)
http_request.return_value = responseObject
query = FindQuery('a.b.c', None, None)
nodes = finder.find_nodes(query)
self.assertEqual(http_request.call_args[0], (
'POST',
'https://127.0.0.1/metrics/find/',
))
self.assertEqual(http_request.call_args[1], {
'fields': [
('local', '1'),
('format', 'msgpack'),
('query', 'a.b.c'),
],
'headers': None,
'preload_content': False,
'timeout': 10,
})
self.assertEqual(len(nodes), 2)
self.assertIsInstance(nodes[0], BranchNode)
self.assertEqual(nodes[0].path, 'a.b.c')
self.assertIsInstance(nodes[1], LeafNode)
self.assertEqual(nodes[1].path, 'a.b.c.d')
# non-pickle response
responseObject = HTTPResponse(body=BytesIO(b'error'), status=200, preload_content=False)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception, 'Error decoding response from https://[^ ]+: .+'):
finder.find_nodes(query)
