def msgpack_nodes(nodes): nodes_info = [] for node in nodes: info = dict(path=node.path, is_leaf=node.is_leaf) if node.is_leaf: info['intervals'] = [interval.tuple for interval in node.intervals] nodes_info.append(info) return msgpack.dumps(nodes_info)
def msgpack_nodes(nodes): nodes_info = [] # make sure everything is unicode in python 2.x and 3.x for node in nodes: info = { text_type('path'): text_type(node.path), text_type('is_leaf'): node.is_leaf, } if node.is_leaf: info[text_type('intervals')] = [interval.tuple for interval in node.intervals] nodes_info.append(info) return msgpack.dumps(nodes_info, use_bin_type=True)
def test_RemoteReader_fetch_multi(self, http_request): test_finders = RemoteFinder.factory() finder = test_finders[0] startTime = 1496262000 endTime = 1496262060 # no path or bulk_query reader = RemoteReader(finder,{}) self.assertEqual(reader.bulk_query, []) result = reader.fetch_multi(startTime, endTime) self.assertEqual(result, []) self.assertEqual(http_request.call_count, 0) # path reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'}) data = [ {'start': startTime, 'step': 60, 'end': endTime, 'values': [1.0, 0.0, 1.0, 0.0, 1.0], 'name': 'a.b.c.d' } ] responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)), status=200, preload_content=False) http_request.return_value = responseObject result = reader.fetch_multi(startTime, endTime) expected_response = [ { 'pathExpression': 'a.b.c.d', 'name': 'a.b.c.d', 'time_info': (1496262000, 1496262060, 60), 'values': [1.0, 0.0, 1.0, 0.0, 1.0], } ] self.assertEqual(result, expected_response) self.assertEqual(http_request.call_args[0], ( 'GET', 'http://127.0.0.1/render/', )) self.assertEqual(http_request.call_args[1], { 'fields': [ ('format', 'pickle'), ('local', '1'), ('noCache', '1'), ('from', startTime), ('until', endTime), ('target', 'a.b.c.d'), ], 'headers': None, 'preload_content': False, 'timeout': 10, }) # bulk_query & now finder = test_finders[1] reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'}, bulk_query=['a.b.c.d']) data = [ {'start': startTime, 'step': 60, 'end': endTime, 'values': [1.0, 0.0, 1.0, 0.0, 1.0], 'name': 'a.b.c.d' } ] responseObject = HTTPResponse( body=BytesIO(msgpack.dumps(data, use_bin_type=True)), status=200, preload_content=False, headers={'Content-Type': 'application/x-msgpack'} ) http_request.return_value = responseObject result = reader.fetch_multi(startTime, endTime, now=endTime, requestContext={'forwardHeaders': {'Authorization': 'Basic xxxx'}}) expected_response = [ { 'pathExpression': 'a.b.c.d', 'name': 'a.b.c.d', 'time_info': (1496262000, 1496262060, 60), 'values': [1.0, 0.0, 1.0, 0.0, 1.0], } ] self.assertEqual(result, expected_response) self.assertEqual(http_request.call_args[0], ( 'GET', 'http://8.8.8.8/graphite/render/', )) self.assertEqual(http_request.call_args[1], { 'fields': [ ('format', 'msgpack'), ('local', '0'), ('noCache', '1'), ('from', startTime), ('until', endTime), ('target', 'a.b.c.d'), ('now', endTime), ], 'headers': {'Authorization': 'Basic xxxx'}, 'preload_content': False, 'timeout': 10, }) # non-pickle response responseObject = HTTPResponse(body=BytesIO(b'error'), status=200, preload_content=False) http_request.return_value = responseObject with self.assertRaisesRegexp(Exception, 'Error decoding response from http://[^ ]+: .+'): reader.fetch(startTime, endTime) # invalid response data data = [ {}, ] responseObject = HTTPResponse( body=BytesIO(msgpack.dumps(data, use_bin_type=True)), status=200, preload_content=False, headers={'Content-Type': 'application/x-msgpack'} ) http_request.return_value = responseObject with self.assertRaisesRegexp(Exception, 'Invalid render response from http://[^ ]+: KeyError\(\'name\',?\)'): reader.fetch(startTime, endTime) # non-200 response responseObject = HTTPResponse(body=BytesIO(b'error'), status=500, preload_content=False) http_request.return_value = responseObject with self.assertRaisesRegexp(Exception, 'Error response 500 from http://[^ ]+'): reader.fetch(startTime, endTime) # exception raised by request() http_request.side_effect = Exception('error') with self.assertRaisesRegexp(Exception, 'Error requesting http://[^ ]+: error'): reader.fetch(startTime, endTime)
def test_RemoteReader_fetch_multi(self, http_request): test_finders = RemoteFinder.factory() finder = test_finders[0] startTime = 1496262000 endTime = 1496262060 # no path or bulk_query reader = RemoteReader(finder, {}) self.assertEqual(reader.bulk_query, []) result = reader.fetch_multi(startTime, endTime) self.assertEqual(result, []) self.assertEqual(http_request.call_count, 0) # path reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'}) data = [{ 'start': startTime, 'step': 60, 'end': endTime, 'values': [1.0, 0.0, 1.0, 0.0, 1.0], 'name': 'a.b.c.d' }] responseObject = HTTPResponse(body=StringIO(pickle.dumps(data)), status=200, preload_content=False) http_request.return_value = responseObject result = reader.fetch_multi(startTime, endTime) expected_response = [{ 'pathExpression': 'a.b.c.d', 'name': 'a.b.c.d', 'time_info': (1496262000, 1496262060, 60), 'values': [1.0, 0.0, 1.0, 0.0, 1.0], }] self.assertEqual(result, expected_response) self.assertEqual(http_request.call_args[0], ( 'GET', 'http://127.0.0.1/render/', )) self.assertEqual( http_request.call_args[1], { 'fields': [ ('format', 'pickle'), ('local', '1'), ('noCache', '1'), ('from', startTime), ('until', endTime), ('target', 'a.b.c.d'), ], 'headers': None, 'preload_content': False, 'timeout': 10, }) # bulk_query & now finder = test_finders[1] reader = RemoteReader(finder, { 'intervals': [], 'path': 'a.b.c.d' }, bulk_query=['a.b.c.d']) data = [{ 'start': startTime, 'step': 60, 'end': endTime, 'values': [1.0, 0.0, 1.0, 0.0, 1.0], 'name': 'a.b.c.d' }] responseObject = HTTPResponse( body=StringIO(msgpack.dumps(data)), status=200, preload_content=False, headers={'Content-Type': 'application/x-msgpack'}) http_request.return_value = responseObject result = reader.fetch_multi( startTime, endTime, now=endTime, requestContext={'forwardHeaders': { 'Authorization': 'Basic xxxx' }}) expected_response = [{ 'pathExpression': 'a.b.c.d', 'name': 'a.b.c.d', 'time_info': (1496262000, 1496262060, 60), 'values': [1.0, 0.0, 1.0, 0.0, 1.0], }] self.assertEqual(result, expected_response) self.assertEqual(http_request.call_args[0], ( 'GET', 'http://8.8.8.8/graphite/render/', )) self.assertEqual( http_request.call_args[1], { 'fields': [ ('format', 'msgpack'), ('local', '0'), ('noCache', '1'), ('from', startTime), ('until', endTime), ('target', 'a.b.c.d'), ('now', endTime), ], 'headers': { 'Authorization': 'Basic xxxx' }, 'preload_content': False, 'timeout': 10, }) # non-pickle response responseObject = HTTPResponse(body=StringIO('error'), status=200, preload_content=False) http_request.return_value = responseObject with self.assertRaisesRegexp( Exception, 'Error decoding render response from http://[^ ]+: .+'): reader.fetch(startTime, endTime) # non-200 response responseObject = HTTPResponse(body=StringIO('error'), status=500, preload_content=False) http_request.return_value = responseObject with self.assertRaisesRegexp(Exception, 'Error response 500 from http://[^ ]+'): reader.fetch(startTime, endTime) # exception raised by request() http_request.side_effect = Exception('error') with self.assertRaisesRegexp(Exception, 'Error requesting http://[^ ]+: error'): reader.fetch(startTime, endTime)
def test_find_nodes(self, http_request): finder = RemoteFinder('127.0.0.1') startTime = 1496262000 endTime = 1496262060 data = [ { 'path': 'a.b.c', 'is_leaf': False, }, { 'path': 'a.b.c.d', 'is_leaf': True, }, ] responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)), status=200, preload_content=False) http_request.return_value = responseObject query = FindQuery('a.b.c', startTime, endTime) result = finder.find_nodes(query) self.assertIsInstance(result, types.GeneratorType) nodes = list(result) self.assertEqual(http_request.call_args[0], ( 'POST', 'http://127.0.0.1/metrics/find/', )) self.assertEqual( http_request.call_args[1], { 'fields': [ ('local', '1'), ('format', 'pickle'), ('query', 'a.b.c'), ('from', startTime), ('until', endTime), ], 'headers': None, 'preload_content': False, 'timeout': 10, }) self.assertEqual(len(nodes), 2) self.assertIsInstance(nodes[0], BranchNode) self.assertEqual(nodes[0].path, 'a.b.c') self.assertIsInstance(nodes[1], LeafNode) self.assertEqual(nodes[1].path, 'a.b.c.d') finder = RemoteFinder('https://127.0.0.1?format=msgpack') data = [ { 'path': 'a.b.c', 'is_leaf': False, }, { 'path': 'a.b.c.d', 'is_leaf': True, }, ] responseObject = HTTPResponse( body=BytesIO(msgpack.dumps(data, use_bin_type=True)), status=200, preload_content=False, headers={'Content-Type': 'application/x-msgpack'}) http_request.return_value = responseObject query = FindQuery('a.b.c', None, None) result = finder.find_nodes(query) self.assertIsInstance(result, types.GeneratorType) nodes = list(result) self.assertEqual(http_request.call_args[0], ( 'POST', 'https://127.0.0.1/metrics/find/', )) self.assertEqual( http_request.call_args[1], { 'fields': [ ('local', '1'), ('format', 'msgpack'), ('query', 'a.b.c'), ], 'headers': None, 'preload_content': False, 'timeout': 10, }) self.assertEqual(len(nodes), 2) self.assertIsInstance(nodes[0], BranchNode) self.assertEqual(nodes[0].path, 'a.b.c') self.assertIsInstance(nodes[1], LeafNode) self.assertEqual(nodes[1].path, 'a.b.c.d') # non-pickle response responseObject = HTTPResponse(body=BytesIO(b'error'), status=200, preload_content=False) http_request.return_value = responseObject result = finder.find_nodes(query) with self.assertRaisesRegexp( Exception, 'Error decoding find response from https://[^ ]+: .+'): list(result)
def _test_find_nodes(self, http_request): finder = RemoteFinder('127.0.0.1') startTime = 1496262000 endTime = 1496262060 data = [ { 'path': 'a.b.c', 'is_leaf': False, }, { 'path': 'a.b.c.d', 'is_leaf': True, }, ] responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)), status=200, preload_content=False) http_request.return_value = responseObject query = FindQuery('a.b.c', startTime, endTime) nodes = finder.find_nodes(query) self.assertEqual(http_request.call_args[0], ( 'POST', 'http://127.0.0.1/metrics/find/', )) self.assertEqual(http_request.call_args[1], { 'fields': [ ('local', '1'), ('format', 'pickle'), ('query', 'a.b.c'), ('from', startTime), ('until', endTime), ], 'headers': None, 'preload_content': False, 'timeout': 10, }) self.assertEqual(len(nodes), 2) self.assertIsInstance(nodes[0], BranchNode) self.assertEqual(nodes[0].path, 'a.b.c') self.assertIsInstance(nodes[1], LeafNode) self.assertEqual(nodes[1].path, 'a.b.c.d') finder = RemoteFinder('https://127.0.0.1?format=msgpack') data = [ { 'path': 'a.b.c', 'is_leaf': False, }, { 'path': 'a.b.c.d', 'is_leaf': True, }, ] responseObject = HTTPResponse( body=BytesIO(msgpack.dumps(data, use_bin_type=True)), status=200, preload_content=False, headers={'Content-Type': 'application/x-msgpack'} ) http_request.return_value = responseObject query = FindQuery('a.b.c', None, None) nodes = finder.find_nodes(query) self.assertEqual(http_request.call_args[0], ( 'POST', 'https://127.0.0.1/metrics/find/', )) self.assertEqual(http_request.call_args[1], { 'fields': [ ('local', '1'), ('format', 'msgpack'), ('query', 'a.b.c'), ], 'headers': None, 'preload_content': False, 'timeout': 10, }) self.assertEqual(len(nodes), 2) self.assertIsInstance(nodes[0], BranchNode) self.assertEqual(nodes[0].path, 'a.b.c') self.assertIsInstance(nodes[1], LeafNode) self.assertEqual(nodes[1].path, 'a.b.c.d') # non-pickle response responseObject = HTTPResponse(body=BytesIO(b'error'), status=200, preload_content=False) http_request.return_value = responseObject with self.assertRaisesRegexp(Exception, 'Error decoding response from https://[^ ]+: .+'): finder.find_nodes(query)