def delegateRendering(graphType, graphOptions, headers=None):
if headers is None:
headers = {}
start = time()
postData = graphType + '\n' + pickle.dumps(graphOptions)
servers = settings.RENDERING_HOSTS[:] #make a copy so we can shuffle it safely
shuffle(servers)
connector_class = connector_class_selector(settings.INTRACLUSTER_HTTPS)
for server in servers:
start2 = time()
try:
# Get a connection
try:
pool = connectionPools[server]
except KeyError: #happens the first time
pool = connectionPools[server] = set()
try:
connection = pool.pop()
except KeyError: #No available connections, have to make a new one
connection = connector_class(server)
connection.timeout = settings.REMOTE_RENDER_CONNECT_TIMEOUT
# Send the request
try:
connection.request('POST', '/render/local/', postData, headers)
except six.moves.http_client.CannotSendRequest:
connection = connector_class(server) #retry once
connection.timeout = settings.REMOTE_RENDER_CONNECT_TIMEOUT
connection.request('POST', '/render/local/', postData, headers)
# Read the response
try: # Python 2.7+, use buffering of HTTP responses
response = connection.getresponse(buffering=True)
except TypeError: # Python 2.6 and older
response = connection.getresponse()
assert response.status == 200, "Bad response code %d from %s" % (
response.status, server)
contentType = response.getheader('Content-Type')
imageData = response.read()
assert contentType == 'image/png', "Bad content type: \"%s\" from %s" % (
contentType, server)
assert imageData, "Received empty response from %s" % server
# Wrap things up
log.rendering('Remotely rendered image on %s in %.6f seconds' %
(server, time() - start2))
log.rendering(
'Spent a total of %.6f seconds doing remote rendering work' %
(time() - start))
pool.add(connection)
return imageData
except:
log.exception(
"Exception while attempting remote rendering request on %s" %
server)
log.rendering(
'Exception while remotely rendering on %s wasted %.6f' %
(server, time() - start2))
continue
def pickle_nodes(nodes):
nodes_info = []
for node in nodes:
info = dict(path=node.path, is_leaf=node.is_leaf)
if node.is_leaf:
info['intervals'] = node.intervals
nodes_info.append(info)
return pickle.dumps(nodes_info, protocol=-1)
def pickle_nodes(nodes):
nodes_info = []
for node in nodes:
info = dict(path=node.path, is_leaf=node.is_leaf)
if node.is_leaf:
info['intervals'] = node.intervals
nodes_info.append(info)
return pickle.dumps(nodes_info, protocol=-1)
def test_RemoteReader_fetch(self, http_request):
test_finders = RemoteFinder.factory()
finder = test_finders[0]
startTime = 1496262000
endTime = 1496262060
# no path or bulk_query
reader = RemoteReader(finder,{})
self.assertEqual(reader.bulk_query, [])
result = reader.fetch(startTime, endTime)
self.assertEqual(result, None)
self.assertEqual(http_request.call_count, 0)
# path & bulk_query
reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'}, bulk_query=['a.b.c.*'])
data = [
{'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.c'
},
{'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}
]
responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)), status=200, preload_content=False)
http_request.return_value = responseObject
result = reader.fetch(startTime, endTime)
expected_response = ((1496262000, 1496262060, 60), [1.0, 0.0, 1.0, 0.0, 1.0])
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://127.0.0.1/render/',
))
self.assertEqual(http_request.call_args[1], {
'fields': [
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.*'),
],
'headers': None,
'preload_content': False,
'timeout': 10,
})
def test_RemoteFinder_fetch(self, http_request):
test_finders = RemoteFinder.factory()
finder = test_finders[0]
startTime = 1496262000
endTime = 1496262060
data = [
{
'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d',
},
]
responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)),
status=200,
preload_content=False)
http_request.return_value = responseObject
result = finder.fetch(['a.b.c.d'], startTime, endTime)
expected_response = [
{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
},
]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'POST',
'https://127.0.0.1/render/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
],
'headers':
None,
'preload_content':
False,
'timeout':
10,
})
def delegateRendering(graphType, graphOptions, headers=None):
if headers is None:
headers = {}
start = time()
postData = graphType + '\n' + pickle.dumps(graphOptions)
servers = settings.RENDERING_HOSTS[:] #make a copy so we can shuffle it safely
shuffle(servers)
connector_class = connector_class_selector(settings.INTRACLUSTER_HTTPS)
for server in servers:
start2 = time()
try:
# Get a connection
try:
pool = connectionPools[server]
except KeyError: #happens the first time
pool = connectionPools[server] = set()
try:
connection = pool.pop()
except KeyError: #No available connections, have to make a new one
connection = connector_class(server)
connection.timeout = settings.REMOTE_RENDER_CONNECT_TIMEOUT
# Send the request
try:
connection.request('POST','/render/local/', postData, headers)
except six.moves.http_client.CannotSendRequest:
connection = connector_class(server) #retry once
connection.timeout = settings.REMOTE_RENDER_CONNECT_TIMEOUT
connection.request('POST', '/render/local/', postData, headers)
# Read the response
try: # Python 2.7+, use buffering of HTTP responses
response = connection.getresponse(buffering=True)
except TypeError: # Python 2.6 and older
response = connection.getresponse()
assert response.status == 200, "Bad response code %d from %s" % (response.status,server)
contentType = response.getheader('Content-Type')
imageData = response.read()
assert contentType == 'image/png', "Bad content type: \"%s\" from %s" % (contentType,server)
assert imageData, "Received empty response from %s" % server
# Wrap things up
log.rendering('Remotely rendered image on %s in %.6f seconds' % (server,time() - start2))
log.rendering('Spent a total of %.6f seconds doing remote rendering work' % (time() - start))
pool.add(connection)
return imageData
except:
log.exception("Exception while attempting remote rendering request on %s" % server)
log.rendering('Exception while remotely rendering on %s wasted %.6f' % (server,time() - start2))
continue
def test_RemoteFinder_fetch(self, http_request):
test_finders = RemoteFinder.factory()
finder = test_finders[0]
startTime = 1496262000
endTime = 1496262060
data = [
{
'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d',
},
]
responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)), status=200, preload_content=False)
http_request.return_value = responseObject
result = finder.fetch(['a.b.c.d'], startTime, endTime)
expected_response = [
{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
},
]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'POST',
'https://127.0.0.1/render/',
))
self.assertEqual(http_request.call_args[1], {
'fields': [
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
],
'headers': None,
'preload_content': False,
'timeout': 10,
})
def test_RemoteReader_fetch_multi(self, http_request):
test_finders = RemoteFinder.factory()
finder = test_finders[0]
startTime = 1496262000
endTime = 1496262060
# no path or bulk_query
reader = RemoteReader(finder,{})
self.assertEqual(reader.bulk_query, [])
result = reader.fetch_multi(startTime, endTime)
self.assertEqual(result, [])
self.assertEqual(http_request.call_count, 0)
# path
reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'})
data = [
{'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}
]
responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)), status=200, preload_content=False)
http_request.return_value = responseObject
result = reader.fetch_multi(startTime, endTime)
expected_response = [
{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
}
]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://127.0.0.1/render/',
))
self.assertEqual(http_request.call_args[1], {
'fields': [
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
],
'headers': None,
'preload_content': False,
'timeout': 10,
})
# bulk_query & now
finder = test_finders[1]
reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'}, bulk_query=['a.b.c.d'])
data = [
{'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}
]
responseObject = HTTPResponse(
body=BytesIO(msgpack.dumps(data, use_bin_type=True)),
status=200,
preload_content=False,
headers={'Content-Type': 'application/x-msgpack'}
)
http_request.return_value = responseObject
result = reader.fetch_multi(startTime, endTime, now=endTime, requestContext={'forwardHeaders': {'Authorization': 'Basic xxxx'}})
expected_response = [
{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
}
]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://8.8.8.8/graphite/render/',
))
self.assertEqual(http_request.call_args[1], {
'fields': [
('format', 'msgpack'),
('local', '0'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
('now', endTime),
],
'headers': {'Authorization': 'Basic xxxx'},
'preload_content': False,
'timeout': 10,
})
# non-pickle response
responseObject = HTTPResponse(body=BytesIO(b'error'), status=200, preload_content=False)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception, 'Error decoding response from http://[^ ]+: .+'):
reader.fetch(startTime, endTime)
# invalid response data
data = [
{},
]
responseObject = HTTPResponse(
body=BytesIO(msgpack.dumps(data, use_bin_type=True)),
status=200,
preload_content=False,
headers={'Content-Type': 'application/x-msgpack'}
)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception, 'Invalid render response from http://[^ ]+: KeyError\(\'name\',?\)'):
reader.fetch(startTime, endTime)
# non-200 response
responseObject = HTTPResponse(body=BytesIO(b'error'), status=500, preload_content=False)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception, 'Error response 500 from http://[^ ]+'):
reader.fetch(startTime, endTime)
# exception raised by request()
http_request.side_effect = Exception('error')
with self.assertRaisesRegexp(Exception, 'Error requesting http://[^ ]+: error'):
reader.fetch(startTime, endTime)
def test_RemoteReader_fetch_multi(self, http_request):
test_finders = RemoteFinder.factory()
finder = test_finders[0]
startTime = 1496262000
endTime = 1496262060
# no path or bulk_query
reader = RemoteReader(finder, {})
self.assertEqual(reader.bulk_query, [])
result = reader.fetch_multi(startTime, endTime)
self.assertEqual(result, [])
self.assertEqual(http_request.call_count, 0)
# path
reader = RemoteReader(finder, {'intervals': [], 'path': 'a.b.c.d'})
data = [{
'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}]
responseObject = HTTPResponse(body=StringIO(pickle.dumps(data)),
status=200,
preload_content=False)
http_request.return_value = responseObject
result = reader.fetch_multi(startTime, endTime)
expected_response = [{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
}]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://127.0.0.1/render/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
],
'headers':
None,
'preload_content':
False,
'timeout':
10,
})
# bulk_query & now
finder = test_finders[1]
reader = RemoteReader(finder, {
'intervals': [],
'path': 'a.b.c.d'
},
bulk_query=['a.b.c.d'])
data = [{
'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}]
responseObject = HTTPResponse(
body=StringIO(msgpack.dumps(data)),
status=200,
preload_content=False,
headers={'Content-Type': 'application/x-msgpack'})
http_request.return_value = responseObject
result = reader.fetch_multi(
startTime,
endTime,
now=endTime,
requestContext={'forwardHeaders': {
'Authorization': 'Basic xxxx'
}})
expected_response = [{
'pathExpression': 'a.b.c.d',
'name': 'a.b.c.d',
'time_info': (1496262000, 1496262060, 60),
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
}]
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://8.8.8.8/graphite/render/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('format', 'msgpack'),
('local', '0'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.d'),
('now', endTime),
],
'headers': {
'Authorization': 'Basic xxxx'
},
'preload_content':
False,
'timeout':
10,
})
# non-pickle response
responseObject = HTTPResponse(body=StringIO('error'),
status=200,
preload_content=False)
http_request.return_value = responseObject
with self.assertRaisesRegexp(
Exception,
'Error decoding render response from http://[^ ]+: .+'):
reader.fetch(startTime, endTime)
# non-200 response
responseObject = HTTPResponse(body=StringIO('error'),
status=500,
preload_content=False)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception,
'Error response 500 from http://[^ ]+'):
reader.fetch(startTime, endTime)
# exception raised by request()
http_request.side_effect = Exception('error')
with self.assertRaisesRegexp(Exception,
'Error requesting http://[^ ]+: error'):
reader.fetch(startTime, endTime)
def test_RemoteReader_fetch(self, http_request):
test_finders = RemoteFinder.factory()
finder = test_finders[0]
startTime = 1496262000
endTime = 1496262060
# no path or bulk_query
reader = RemoteReader(finder, {})
self.assertEqual(reader.bulk_query, [])
result = reader.fetch(startTime, endTime)
self.assertEqual(result, None)
self.assertEqual(http_request.call_count, 0)
# path & bulk_query
reader = RemoteReader(finder, {
'intervals': [],
'path': 'a.b.c.d'
},
bulk_query=['a.b.c.*'])
data = [{
'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.c'
}, {
'start': startTime,
'step': 60,
'end': endTime,
'values': [1.0, 0.0, 1.0, 0.0, 1.0],
'name': 'a.b.c.d'
}]
responseObject = HTTPResponse(body=StringIO(pickle.dumps(data)),
status=200,
preload_content=False)
http_request.return_value = responseObject
result = reader.fetch(startTime, endTime)
expected_response = ((1496262000, 1496262060, 60),
[1.0, 0.0, 1.0, 0.0, 1.0])
self.assertEqual(result, expected_response)
self.assertEqual(http_request.call_args[0], (
'GET',
'http://127.0.0.1/render/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', startTime),
('until', endTime),
('target', 'a.b.c.*'),
],
'headers':
None,
'preload_content':
False,
'timeout':
10,
})
def test_find_nodes(self, http_request):
finder = RemoteFinder('127.0.0.1')
startTime = 1496262000
endTime = 1496262060
data = [
{
'path': 'a.b.c',
'is_leaf': False,
},
{
'path': 'a.b.c.d',
'is_leaf': True,
},
]
responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)),
status=200,
preload_content=False)
http_request.return_value = responseObject
query = FindQuery('a.b.c', startTime, endTime)
result = finder.find_nodes(query)
self.assertIsInstance(result, types.GeneratorType)
nodes = list(result)
self.assertEqual(http_request.call_args[0], (
'POST',
'http://127.0.0.1/metrics/find/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('local', '1'),
('format', 'pickle'),
('query', 'a.b.c'),
('from', startTime),
('until', endTime),
],
'headers':
None,
'preload_content':
False,
'timeout':
10,
})
self.assertEqual(len(nodes), 2)
self.assertIsInstance(nodes[0], BranchNode)
self.assertEqual(nodes[0].path, 'a.b.c')
self.assertIsInstance(nodes[1], LeafNode)
self.assertEqual(nodes[1].path, 'a.b.c.d')
finder = RemoteFinder('https://127.0.0.1?format=msgpack')
data = [
{
'path': 'a.b.c',
'is_leaf': False,
},
{
'path': 'a.b.c.d',
'is_leaf': True,
},
]
responseObject = HTTPResponse(
body=BytesIO(msgpack.dumps(data, use_bin_type=True)),
status=200,
preload_content=False,
headers={'Content-Type': 'application/x-msgpack'})
http_request.return_value = responseObject
query = FindQuery('a.b.c', None, None)
result = finder.find_nodes(query)
self.assertIsInstance(result, types.GeneratorType)
nodes = list(result)
self.assertEqual(http_request.call_args[0], (
'POST',
'https://127.0.0.1/metrics/find/',
))
self.assertEqual(
http_request.call_args[1], {
'fields': [
('local', '1'),
('format', 'msgpack'),
('query', 'a.b.c'),
],
'headers':
None,
'preload_content':
False,
'timeout':
10,
})
self.assertEqual(len(nodes), 2)
self.assertIsInstance(nodes[0], BranchNode)
self.assertEqual(nodes[0].path, 'a.b.c')
self.assertIsInstance(nodes[1], LeafNode)
self.assertEqual(nodes[1].path, 'a.b.c.d')
# non-pickle response
responseObject = HTTPResponse(body=BytesIO(b'error'),
status=200,
preload_content=False)
http_request.return_value = responseObject
result = finder.find_nodes(query)
with self.assertRaisesRegexp(
Exception,
'Error decoding find response from https://[^ ]+: .+'):
list(result)
def _test_find_nodes(self, http_request):
finder = RemoteFinder('127.0.0.1')
startTime = 1496262000
endTime = 1496262060
data = [
{
'path': 'a.b.c',
'is_leaf': False,
},
{
'path': 'a.b.c.d',
'is_leaf': True,
},
]
responseObject = HTTPResponse(body=BytesIO(pickle.dumps(data)), status=200, preload_content=False)
http_request.return_value = responseObject
query = FindQuery('a.b.c', startTime, endTime)
nodes = finder.find_nodes(query)
self.assertEqual(http_request.call_args[0], (
'POST',
'http://127.0.0.1/metrics/find/',
))
self.assertEqual(http_request.call_args[1], {
'fields': [
('local', '1'),
('format', 'pickle'),
('query', 'a.b.c'),
('from', startTime),
('until', endTime),
],
'headers': None,
'preload_content': False,
'timeout': 10,
})
self.assertEqual(len(nodes), 2)
self.assertIsInstance(nodes[0], BranchNode)
self.assertEqual(nodes[0].path, 'a.b.c')
self.assertIsInstance(nodes[1], LeafNode)
self.assertEqual(nodes[1].path, 'a.b.c.d')
finder = RemoteFinder('https://127.0.0.1?format=msgpack')
data = [
{
'path': 'a.b.c',
'is_leaf': False,
},
{
'path': 'a.b.c.d',
'is_leaf': True,
},
]
responseObject = HTTPResponse(
body=BytesIO(msgpack.dumps(data, use_bin_type=True)),
status=200,
preload_content=False,
headers={'Content-Type': 'application/x-msgpack'}
)
http_request.return_value = responseObject
query = FindQuery('a.b.c', None, None)
nodes = finder.find_nodes(query)
self.assertEqual(http_request.call_args[0], (
'POST',
'https://127.0.0.1/metrics/find/',
))
self.assertEqual(http_request.call_args[1], {
'fields': [
('local', '1'),
('format', 'msgpack'),
('query', 'a.b.c'),
],
'headers': None,
'preload_content': False,
'timeout': 10,
})
self.assertEqual(len(nodes), 2)
self.assertIsInstance(nodes[0], BranchNode)
self.assertEqual(nodes[0].path, 'a.b.c')
self.assertIsInstance(nodes[1], LeafNode)
self.assertEqual(nodes[1].path, 'a.b.c.d')
# non-pickle response
responseObject = HTTPResponse(body=BytesIO(b'error'), status=200, preload_content=False)
http_request.return_value = responseObject
with self.assertRaisesRegexp(Exception, 'Error decoding response from https://[^ ]+: .+'):
finder.find_nodes(query)
