def set_metadata_view(request):
results = {}
if request.method == "GET":
metric = request.GET["metric"]
key = request.GET["key"]
value = request.GET["value"]
try:
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.set_metadata(%s, %s)" % (metric, key))
elif request.method == "POST":
if request.META.get("CONTENT_TYPE") == "application/json":
operations = json.loads(request.body)
else:
operations = json.loads(request.POST["operations"])
for op in operations:
metric = None
try:
metric, key, value = op["metric"], op["key"], op["value"]
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
if metric:
results[metric] = dict(
error="Unexpected error occurred in bulk CarbonLink.set_metadata(%s)" % metric
)
else:
results = dict(error="Invalid request method")
return json_response_for(request, results)
def test_set_metadata_view(self):
"""Stub to test set_metadata_view. This currently doesn't test a valid key """
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
url = reverse('metrics_set_metadata')
# GET
# bad key
request = {'metric': 'hosts.worker1.cpu', 'key': 'a', 'value': 'b'}
response = self.client.get(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(
data['hosts.worker1.cpu']['error'],
"Unexpected error occurred in CarbonLink.set_metadata(hosts.worker1.cpu, a)"
)
# POST
# bad key
request = {
'operations':
'[{ "metric": "hosts.worker1.cpu", "key": "a", "value": "b" }]'
}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(
data['hosts.worker1.cpu']['error'],
"Unexpected error occurred in bulk CarbonLink.set_metadata(hosts.worker1.cpu)"
)
def request(self, method, url, fields, requestContext=None):
headers = requestContext.get('forwardHeaders') if requestContext else {}
if 'Authorization' not in headers and self.username and self.password:
user_pw = '%s:%s' % (self.username, self.password)
if sys.version_info[0] >= 3:
user_pw_b64 = b2a_base64(user_pw.encode('utf-8')).decode('ascii')
else:
user_pw_b64 = user_pw.encode('base64')
headers['Authorization'] = 'Basic ' + user_pw_b64
req_fields = []
for (field, value) in fields.items():
if value is None:
continue
if isinstance(value, list) or isinstance(value, tuple):
req_fields.extend([(field, v) for v in value if v is not None])
else:
req_fields.append((field, value))
result = http.request(
method,
self.base_url + url,
fields=req_fields,
headers=headers,
timeout=self.settings.FIND_TIMEOUT,
)
if result.status == 400:
raise ValueError(json.loads(result.data.decode('utf-8')).get('error'))
if result.status != 200:
raise Exception('HTTP Error from remote tagdb: %s %s' % (result.status, result.data))
return json.loads(result.data.decode('utf-8'))
def request(self, method, url, fields, requestContext=None):
headers = requestContext.get('forwardHeaders') if requestContext else {}
if 'Authorization' not in headers and self.username and self.password:
user_pw = '%s:%s' % (self.username, self.password)
if sys.version_info[0] >= 3:
user_pw_b64 = b2a_base64(user_pw.encode('utf-8')).decode('ascii')
else:
user_pw_b64 = user_pw.encode('base64')
headers['Authorization'] = 'Basic ' + user_pw_b64
req_fields = []
for (field, value) in fields.items():
if value is None:
continue
if isinstance(value, list) or isinstance(value, tuple):
req_fields.extend([(field, v) for v in value if v is not None])
else:
req_fields.append((field, value))
result = http.request(
method,
self.base_url + url,
fields=req_fields,
headers=headers,
timeout=self.settings.REMOTE_FIND_TIMEOUT,
)
if result.status == 400:
raise ValueError(json.loads(result.data.decode('utf-8')).get('error'))
if result.status != 200:
raise Exception('HTTP Error from remote tagdb: %s %s' % (result.status, result.data))
return json.loads(result.data.decode('utf-8'))
def test_timezone_handling(self):
url = reverse('events')
data = {'what': 'something happened', 'when': time.time() - 3590}
with self.settings(TIME_ZONE='Europe/Moscow'):
response = self.client.post(url,
json.dumps(data),
content_type='application/json')
self.assertEqual(response.status_code, 200)
event = Event.objects.get()
self.assertTrue(timezone.now() - event.when < timedelta(hours=1))
self.assertTrue(timezone.now() - event.when > timedelta(seconds=3580))
url = reverse('events_get_data')
with self.settings(TIME_ZONE='Europe/Berlin'):
response1 = self.client.get(url)
[event] = json.loads(response1.content)
self.assertEqual(event['what'], 'something happened')
with self.settings(TIME_ZONE='UTC'):
response2 = self.client.get(url)
self.assertEqual(response1.content, response2.content)
url = reverse('events')
data = {'what': 'something else happened'}
with self.settings(TIME_ZONE='Asia/Hong_Kong'):
response = self.client.post(url,
json.dumps(data),
content_type='application/json')
self.assertEqual(response.status_code, 200)
url = reverse('events_get_data')
with self.settings(TIME_ZONE='Europe/Berlin'):
response = self.client.get(url, {'from': int(time.time() - 3500)})
[event] = json.loads(response.content)
self.assertEqual(event['what'], 'something else happened')
def test_get_detail_json_html(self):
creation_url = reverse('events')
event = {
'what': 'Something happened',
'data': 'more info',
'tags': ['foo', 'bar', 'baz'],
}
response = self.client.post(creation_url,
json.dumps(event),
content_type='application/json')
self.assertEqual(response.status_code, 200)
url = reverse('events_get_data')
# should match two events using old set logic
response = self.client.get(url, {'tags': 'foo bar baz'})
self.assertEqual(response.status_code, 200)
events = json.loads(response.content)
self.assertEqual(len(events), 1)
url = reverse('events_detail', args=[events[0]['id']])
response = self.client.get(url, {}, HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, 200)
event = json.loads(response.content)
self.assertEqual(event['what'], 'Something happened')
self.assertEqual(event['tags'], ['foo', 'bar', 'baz'])
url = reverse('events_detail', args=[events[0]['id']])
response = self.client.get(url, {})
self.assertEqual(response.status_code, 200)
self.assertRegexpMatches(
response.content,
re.compile(
b'^<html>.+<title>Something happened</title>.+<h1>Something happened</h1>.+<td>tags</td><td>foo bar baz</td>.+</html>$',
re.DOTALL))
def set_metadata_view(request):
results = {}
if request.method == 'GET':
metric = request.GET['metric']
key = request.GET['key']
value = request.GET['value']
try:
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.set_metadata(%s, %s)" % (metric, key))
elif request.method == 'POST':
if request.META.get('CONTENT_TYPE') == 'application/json':
operations = json.loads( request.body )
else:
operations = json.loads( request.POST['operations'] )
for op in operations:
metric = None
try:
metric, key, value = op['metric'], op['key'], op['value']
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
if metric:
results[metric] = dict(error="Unexpected error occurred in bulk CarbonLink.set_metadata(%s)" % metric)
else:
results = dict(error="Invalid request method")
return json_response_for(request, results)
def test_event_tags(self):
url = reverse('events_get_data')
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(json.loads(response.content)), 0)
creation_url = reverse('events')
event = {
'what': 'Something happened',
'data': 'more info',
'tags': ['foo', 'bar'],
}
response = self.client.post(creation_url,
json.dumps(event),
content_type='application/json')
self.assertEqual(response.status_code, 200)
event = Event.objects.get()
self.assertEqual(event.what, 'Something happened')
self.assertEqual(event.tags, 'foo bar')
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
events = json.loads(response.content)
self.assertEqual(len(events), 1)
event = events[0]
self.assertEqual(event['what'], 'Something happened')
self.assertEqual(event['tags'], ['foo', 'bar'])
response = self.client.get(creation_url)
self.assertEqual(response.status_code, 200)
expected_re = b'^<html>.+<title>Events</title>.+Something happened.+<td>\\['foo', 'bar'\\]</td>.+</html>$'
self.assertRegexpMatches(response.content,
re.compile(expected_re, re.DOTALL))
def test_dashboard_login_invalid_authenticate(self):
url = reverse('dashboard_login')
request = {"username": "******",
"password": "******"}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
self.assertEqual(json.loads(response.content), json.loads('{"errors": {"reason": "Username and/or password invalid."}, "success": false, "text": {}, "permissions": []}'))
def set_metadata_view(request):
results = {}
if request.method == 'GET':
metric = request.GET['metric']
key = request.GET['key']
value = request.GET['value']
try:
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.set_metadata(%s, %s)" % (metric, key))
elif request.method == 'POST':
if request.META.get('CONTENT_TYPE') == 'application/json':
operations = json.loads( request.body )
else:
operations = json.loads( request.POST['operations'] )
for op in operations:
metric = None
try:
metric, key, value = op['metric'], op['key'], op['value']
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
if metric:
results[metric] = dict(error="Unexpected error occurred in bulk CarbonLink.set_metadata(%s)" % metric)
else:
results = dict(error="Invalid request method")
return json_response_for(request, results)
def test_dashboard_logout(self):
url = reverse('dashboard_logout')
request = {"username": "******"}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
self.assertEqual(
json.loads(response.content),
json.loads('{"errors": {}, "success": true, "text": {}}'))
def test_tag_views(self):
url = reverse('tagList')
expected = 'test.a;blah=blah;hello=tiger'
response = self.client.post(url + '/tagSeries', {'path': 'test.a;hello=tiger;blah=blah'})
self.assertEqual(response['Content-Type'], 'application/json')
self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True))
expected = [{"tag": "hello"}]
response = self.client.get(url, {'filter': 'hello$'})
self.assertEqual(response['Content-Type'], 'application/json')
result = json.loads(response.content)
self.assertEqual(len(result), len(expected))
self.assertEqual(result[0]['tag'], expected[0]['tag'])
response = self.client.get(url, {'filter': 'hello$', 'pretty': 1})
self.assertEqual(response['Content-Type'], 'application/json')
result = json.loads(response.content)
self.assertEqual(len(result), len(expected))
self.assertEqual(result[0]['tag'], expected[0]['tag'])
expected = {"tag": "hello", "values": [{"count": 1, "value": "tiger"}]}
response = self.client.get(url + '/hello', {'filter': 'tiger$'})
self.assertEqual(response['Content-Type'], 'application/json')
result = json.loads(response.content)
self.assertEqual(result['tag'], expected['tag'])
self.assertEqual(len(result['values']), len(expected['values']))
self.assertEqual(result['values'][0]['count'], expected['values'][0]['count'])
self.assertEqual(result['values'][0]['value'], expected['values'][0]['value'])
response = self.client.get(url + '/hello', {'filter': 'tiger$', 'pretty': 1})
self.assertEqual(response['Content-Type'], 'application/json')
result = json.loads(response.content)
self.assertEqual(result['tag'], expected['tag'])
self.assertEqual(len(result['values']), len(expected['values']))
self.assertEqual(result['values'][0]['count'], expected['values'][0]['count'])
self.assertEqual(result['values'][0]['value'], expected['values'][0]['value'])
expected = ['test.a;blah=blah;hello=tiger']
response = self.client.get(url + '/findSeries?expr[]=name=test.a&expr[]=hello=tiger&expr[]=blah=blah&pretty=1')
self.assertEqual(response['Content-Type'], 'application/json')
self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True))
expected = True
response = self.client.post(url + '/delSeries', {'path': 'test.a;blah=blah;hello=tiger'})
self.assertEqual(response['Content-Type'], 'application/json')
self.assertEqual(response.content, json.dumps(expected))
expected = []
response = self.client.get(url + '/findSeries?expr=name=test.a&expr=hello=tiger&expr=blah=blah')
self.assertEqual(response['Content-Type'], 'application/json')
self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True))
def test_tag_views(self):
url = reverse('tagList')
expected = 'test.a;blah=blah;hello=tiger'
response = self.client.post(url + '/tagSeries', {'path': 'test.a;hello=tiger;blah=blah'})
self.assertEqual(response['Content-Type'], 'application/json')
self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True))
expected = [{"tag": "hello"}]
response = self.client.get(url, {'filter': 'hello$'})
self.assertEqual(response['Content-Type'], 'application/json')
result = json.loads(response.content)
self.assertEqual(len(result), len(expected))
self.assertEqual(result[0]['tag'], expected[0]['tag'])
response = self.client.get(url, {'filter': 'hello$', 'pretty': 1})
self.assertEqual(response['Content-Type'], 'application/json')
result = json.loads(response.content)
self.assertEqual(len(result), len(expected))
self.assertEqual(result[0]['tag'], expected[0]['tag'])
expected = {"tag": "hello", "values": [{"count": 1, "value": "tiger"}]}
response = self.client.get(url + '/hello', {'filter': 'tiger$'})
self.assertEqual(response['Content-Type'], 'application/json')
result = json.loads(response.content)
self.assertEqual(result['tag'], expected['tag'])
self.assertEqual(len(result['values']), len(expected['values']))
self.assertEqual(result['values'][0]['count'], expected['values'][0]['count'])
self.assertEqual(result['values'][0]['value'], expected['values'][0]['value'])
response = self.client.get(url + '/hello', {'filter': 'tiger$', 'pretty': 1})
self.assertEqual(response['Content-Type'], 'application/json')
result = json.loads(response.content)
self.assertEqual(result['tag'], expected['tag'])
self.assertEqual(len(result['values']), len(expected['values']))
self.assertEqual(result['values'][0]['count'], expected['values'][0]['count'])
self.assertEqual(result['values'][0]['value'], expected['values'][0]['value'])
expected = ['test.a;blah=blah;hello=tiger']
response = self.client.get(url + '/findSeries?expr[]=name=test.a&expr[]=hello=tiger&expr[]=blah=blah&pretty=1')
self.assertEqual(response['Content-Type'], 'application/json')
self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True))
expected = True
response = self.client.post(url + '/delSeries', {'path': 'test.a;blah=blah;hello=tiger'})
self.assertEqual(response['Content-Type'], 'application/json')
self.assertEqual(response.content, json.dumps(expected))
expected = []
response = self.client.get(url + '/findSeries?expr=name=test.a&expr=hello=tiger&expr=blah=blah')
self.assertEqual(response['Content-Type'], 'application/json')
self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True))
def test_dashboard_login_valid_authenticate(self, authenticate):
url = reverse('dashboard_login')
request = {"username": "******",
"password": "******"}
user = User.objects.create(email='*****@*****.**')
user.backend = ''
authenticate.return_value = user
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
self.assertEqual(json.loads(response.content), json.loads('{"permissions": ["change", "delete"], "success": true, "text": {}, "errors": {}}'))
def test_dashboard_login_invalid_authenticate(self):
url = reverse('dashboard_login')
request = {"username": "******", "password": "******"}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
self.assertEqual(
json.loads(response.content),
json.loads(
'{"errors": {"reason": "Username and/or password invalid."}, "success": false, "text": {}, "permissions": []}'
))
def test_dashboard_login_valid_authenticate_not_active(self, authenticate):
url = reverse('dashboard_login')
request = {"username": "******",
"password": "******"}
user = User.objects.create(email='*****@*****.**')
user.backend = ''
user.is_active = False
authenticate.return_value = user
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
self.assertEqual(json.loads(response.content), json.loads('{"permissions": [], "success": false, "errors": {"reason": "Account disabled."}, "text": {}}'))
def test_dashboard_load_template_existing(self):
url = reverse('dashboard_save_template', args=['testtemplate', 'testkey'])
request = copy.deepcopy(self.testtemplate)
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
url = reverse('dashboard_load_template', args=['testtemplate', 'testkey'])
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
graph_data = json.loads(self.testtemplate["state"].replace('__VALUE__', 'testkey'))
self.assertEqual(data, json.loads('{"state": {"name": "testtemplate/testkey", "graphs": ' + json.dumps(graph_data['graphs']) + '}}'))
def test_tag_sets(self):
creation_url = reverse('events')
events = [
{
'what': 'Something happened',
'data': 'more info',
'tags': ['foo'],
},
{
'what': 'Something else happened',
'data': 'even more info',
'tags': ['bar'],
},
{
'what': 'A final thing happened',
'data': 'yet even more info',
'tags': ['foo', 'bar'],
},
]
for event in events:
response = self.client.post(creation_url,
json.dumps(event),
content_type='application/json')
self.assertEqual(response.status_code, 200)
url = reverse('events_get_data')
# should match two events using old set logic
response = self.client.get(url, {'tags': 'foo'})
self.assertEqual(response.status_code, 200)
events = json.loads(response.content)
self.assertEqual(len(events), 2)
# should match one event using set intersection
response = self.client.get("%s%s" %
(url, '?tags=foo%20bar&set=intersection'))
self.assertEqual(response.status_code, 200)
events = json.loads(response.content)
self.assertEqual(len(events), 1)
# should match zero events using set intersection and unknown tag
response = self.client.get(
"%s%s" % (url, '?tags=foo%20bar%20nope&set=intersection'))
self.assertEqual(response.status_code, 200)
events = json.loads(response.content)
self.assertEqual(len(events), 0)
# should match all events using set union
response = self.client.get("%s%s" % (url, '?tags=foo%20bar&set=union'))
self.assertEqual(response.status_code, 200)
events = json.loads(response.content)
self.assertEqual(len(events), 3)
def test_dashboard_login_valid_authenticate(self, authenticate):
url = reverse('dashboard_login')
request = {"username": "******", "password": "******"}
user = User.objects.create(email='*****@*****.**')
user.backend = ''
authenticate.return_value = user
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
self.assertEqual(
json.loads(response.content),
json.loads(
'{"permissions": ["change", "delete"], "success": true, "text": {}, "errors": {}}'
))
def test_dashboard_login_valid_authenticate_not_active(self, authenticate):
url = reverse('dashboard_login')
request = {"username": "******", "password": "******"}
user = User.objects.create(email='*****@*****.**')
user.backend = ''
user.is_active = False
authenticate.return_value = user
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
self.assertEqual(
json.loads(response.content),
json.loads(
'{"permissions": [], "success": false, "errors": {"reason": "Account disabled."}, "text": {}}'
))
def post_event(request):
if request.method == 'POST':
event = json.loads(request.body)
assert isinstance(event, dict)
tags = event.get('tags')
if tags:
if not isinstance(tags, list):
return HttpResponse(
json.dumps({'error': '"tags" must be an array'}),
status=400)
tags = ' '.join(tags)
if 'when' in event:
when = make_aware(
datetime.datetime.utcfromtimestamp(
event.get('when')), pytz.utc)
else:
when = now()
Event.objects.create(
what=event.get('what'),
tags=tags,
when=when,
data=event.get('data', ''),
)
return HttpResponse(status=200)
else:
return HttpResponse(status=405)
def test_unicode_usergraph(self):
url = reverse('browser_usergraph')
user = User.objects.create_user('tèst', '*****@*****.**', 'pass')
self.client.login(username='******', password='******')
self.client.get(
reverse('browser_header')) # this creates a profile for the user
user.profile.mygraph_set.create(name=u'fòo', url='bar')
response = self.client.get(
url, {
'query': 'tèst.*',
'format': 'treejson',
'path': 'tèst',
'user': '******',
'node': 'tèst'
})
self.assertEqual(response.status_code, 200)
[leaf] = json.loads(response.content)
self.assertEqual(
leaf, {
u'leaf': 1,
u'text': u'fòo',
u'allowChildren': 0,
u'graphUrl': u'bar',
u'id': u'tèst.845aa5781192007e1866648eea9f7355',
u'expandable': 0,
})
def post_event(request):
if request.method == 'POST':
event = json.loads(request.body)
assert isinstance(event, dict)
tags = event.get('tags')
if tags is not None:
if isinstance(tags, list):
tags = ' '.join(tags)
elif not isinstance(tags, six.string_types):
return HttpResponse(
json.dumps({'error': '"tags" must be an array or space-separated string'}),
status=400)
else:
tags = None
if 'when' in event:
when = epoch_to_dt(event['when'])
else:
when = now()
Event.objects.create(
what=event.get('what'),
tags=tags,
when=when,
data=event.get('data', ''),
)
return HttpResponse(status=200)
else:
return HttpResponse(status=405)
def index_all_json(request):
if len(settings.CLUSTER_SERVERS) <= 1:
return index_json(request)
matches = []
for cluster_server in settings.CLUSTER_SERVERS:
matches = list(set(matches + json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read())))
return HttpResponse(json.dumps(matches), mimetype='application/json')
def post_event(request):
if request.method == 'POST':
try:
event = json.loads(request.body)
except ValueError:
# try to see if this is a heroku post hook
if "app" in request.POST and "user" in request.POST and "head" in request.POST:
event = {
"what":"%(app)s release" % request.POST,
"tags":"heroku,%(app)s" % request.POST,
"data":"%(app)s released to %(head)s by %(user)s" % request.POST
}
assert isinstance(event, dict)
values = {}
values["what"] = event["what"]
values["tags"] = event.get("tags", None)
values["when"] = datetime.datetime.fromtimestamp(
event.get("when", time.time()))
if "data" in event:
values["data"] = event["data"]
e = models.Event(**values)
e.save()
return HttpResponse(status=200)
else:
return HttpResponse(status=405)
def test_tag_as_str(self):
creation_url = reverse('events')
event = {
'what': 'Something happened',
'data': 'more info',
'tags': 'other',
}
response = self.client.post(creation_url,
json.dumps(event),
content_type='application/json')
self.assertEqual(response.status_code, 200)
event = Event.objects.get()
self.assertEqual(event.what, 'Something happened')
self.assertEqual(event.tags, 'other')
url = reverse('events_get_data')
response = self.client.get(url, {'until': 'now+5min', 'jsonp': 'test'})
self.assertEqual(response.status_code, 200)
self.assertRegexpMatches(response.content, b'^test[(].+[)]$')
events = json.loads(response.content[5:-1])
self.assertEqual(len(events), 1)
event = events[0]
self.assertEqual(event['what'], 'Something happened')
self.assertEqual(event['tags'], ['other'])
def index_json(request):
jsonp = request.REQUEST.get("jsonp", False)
cluster = request.REQUEST.get("cluster", False)
def find_matches():
matches = []
for root, dirs, files in os.walk(settings.WHISPER_DIR):
root = root.replace(settings.WHISPER_DIR, "")
for basename in files:
if fnmatch.fnmatch(basename, "*.wsp"):
matches.append(os.path.join(root, basename))
for root, dirs, files in os.walk(settings.CERES_DIR):
root = root.replace(settings.CERES_DIR, "")
for filename in files:
if filename == ".ceres-node":
matches.append(root)
matches = [m.replace(".wsp", "").replace(".rrd", "").replace("/", ".").lstrip(".") for m in sorted(matches)]
return matches
matches = []
if cluster and len(settings.CLUSTER_SERVERS) > 1:
matches = reduce(
lambda x, y: list(set(x + y)),
[
json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read())
for cluster_server in settings.CLUSTER_SERVERS
],
)
else:
matches = find_matches()
return json_response_for(request, matches, jsonp=jsonp)
def post_event(request):
if request.method == 'POST':
event = json.loads(request.body)
assert isinstance(event, dict)
tags = event.get('tags')
if tags:
if not isinstance(tags, list):
return HttpResponse(json.dumps(
{'error': '"tags" must be an array'}),
status=400)
tags = ' '.join(tags)
if 'when' in event:
when = make_aware(
datetime.datetime.utcfromtimestamp(event.get('when')),
pytz.utc)
else:
when = now()
Event.objects.create(
what=event.get('what'),
tags=tags,
when=when,
data=event.get('data', ''),
)
return HttpResponse(status=200)
else:
return HttpResponse(status=405)
def load(request, name):
try:
dashboard = Dashboard.objects.get(name=name)
except Dashboard.DoesNotExist:
return json_response( dict(error="Dashboard '%s' does not exist. " % name) )
return json_response( dict(state=json.loads(dashboard.state)) )
def load(request, name):
try:
dashboard = Dashboard.objects.get(name=name)
except Dashboard.DoesNotExist:
return json_response( dict(error="Dashboard '%s' does not exist. " % name) )
return json_response( dict(state=json.loads(dashboard.state)) )
def index_json(request):
jsonp = request.REQUEST.get('jsonp', False)
cluster = request.REQUEST.get('cluster', False)
def find_matches():
matches = []
for root, dirs, files in os.walk(settings.WHISPER_DIR):
root = root.replace(settings.WHISPER_DIR, '')
for basename in files:
if fnmatch.fnmatch(basename, '*.wsp'):
matches.append(os.path.join(root, basename))
for root, dirs, files in os.walk(settings.CERES_DIR):
root = root.replace(settings.CERES_DIR, '')
for filename in files:
if filename == '.ceres-node':
matches.append(root)
matches = [
m.replace('.wsp', '').replace('.rrd', '').replace('/',
'.').lstrip('.')
for m in sorted(matches)
]
return matches
matches = []
if cluster and len(settings.CLUSTER_SERVERS) > 1:
matches = reduce( lambda x, y: list(set(x + y)), \
[json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read()) \
for cluster_server in settings.CLUSTER_SERVERS])
else:
matches = find_matches()
return json_response_for(request, matches, jsonp=jsonp)
def post_event(request):
if request.method == 'POST':
event = json.loads(request.body)
assert isinstance(event, dict)
tags = event.get('tags')
if tags is not None:
if isinstance(tags, list):
tags = ' '.join(tags)
elif not isinstance(tags, six.string_types):
return HttpResponse(json.dumps({
'error':
'"tags" must be an array or space-separated string'
}),
status=400)
else:
tags = None
if 'when' in event:
when = epoch_to_dt(event['when'])
else:
when = now()
Event.objects.create(
what=event.get('what'),
tags=tags,
when=when,
data=event.get('data', ''),
)
return HttpResponse(status=200)
else:
return HttpResponse(status=405)
def index_json(request):
queryParams = request.GET.copy()
queryParams.update(request.POST)
jsonp = queryParams.get('jsonp', False)
cluster = queryParams.get('cluster', False)
def find_matches():
matches = []
for root, dirs, files in os.walk(settings.WHISPER_DIR):
root = root.replace(settings.WHISPER_DIR, '')
for basename in files:
if fnmatch.fnmatch(basename, '*.wsp'):
matches.append(os.path.join(root, basename))
for root, dirs, files in os.walk(settings.CERES_DIR):
root = root.replace(settings.CERES_DIR, '')
for filename in files:
if filename == '.ceres-node':
matches.append(root)
# unlike 0.9.x, we're going to use os.walk with followlinks
# since we require Python 2.7 and newer that supports it
if RRDReader.supported:
for root, dirs, files in os.walk(settings.RRD_DIR,
followlinks=True):
root = root.replace(settings.RRD_DIR, '')
for basename in files:
if fnmatch.fnmatch(basename, '*.rrd'):
absolute_path = os.path.join(settings.RRD_DIR, root,
basename)
(basename, extension) = os.path.splitext(basename)
metric_path = os.path.join(root, basename)
rrd = RRDReader(absolute_path, metric_path)
for datasource_name in rrd.get_datasources(
absolute_path):
matches.append(
os.path.join(metric_path, datasource_name))
matches = [
m.replace('.wsp', '').replace('.rrd', '').replace('/',
'.').lstrip('.')
for m in sorted(matches)
]
return matches
matches = []
if cluster and len(settings.CLUSTER_SERVERS) >= 1:
try:
matches = reduce( lambda x, y: list(set(x + y)), \
[json.loads(urllib.urlopen('http://' + cluster_server + '/metrics/index.json').read()) \
for cluster_server in settings.CLUSTER_SERVERS])
except urllib.URLError:
log.exception()
return json_response_for(request, matches, jsonp=jsonp, status=500)
else:
matches = find_matches()
return json_response_for(request, matches, jsonp=jsonp)
def full_path_zon_test(request):
# 0. Parse parameters
queryParams = request.GET.copy()
queryParams.update(request.POST)
# TODO: remove the default values, the default values are what we are using currently
port = queryParams.get('port', '31509')
metric_name = queryParams.get('metric', 'test.fullstack.graphite')
# 1. Send data to graphite
random_data = _send_random_data(metric_name)
# 2. Wait (allow some latency)
time.sleep(1)
# 3. Query graphite
try:
res = urlopen(
"http://localhost:{0}/render/?format=json&target={1}&from=-1min&noCache"
.format(port, metric_name))
s = res.read().decode('utf-8')
json_obj = json.loads(s)
except Exception:
raise
# 4. Check Result
result = ""
details = "Expected value: {0}".format(random_data)
if not json_obj:
result = "fail"
details = "graphite query response is empty"
else:
json_obj = json_obj[0]
if "target" not in json_obj:
result = "fail"
details = "target field is missing"
elif "datapoints" not in json_obj:
result = "fail"
details = "datapoints field is missing"
elif len(json_obj["datapoints"]) != 1:
result = "fail"
details = "the number of returned datapoints is inconsistent"
elif json_obj["datapoints"][0][0] is None:
result = "fail"
details = "datapoint is returned but its value is None"
else:
real_value = int(json_obj["datapoints"][0][0])
expected_value = random_data
if real_value == expected_value:
result = "pass"
else:
result = "fail"
details = "Expected value: {0}, Real Value: {1}".format(
expected_value, real_value)
# 5. Response
result_json_obj = {"result": result, "details": details}
response = HttpResponse(content=json.dumps(result_json_obj),
content_type='application/json')
return response
def index_json(request):
queryParams = request.GET.copy()
queryParams.update(request.POST)
jsonp = queryParams.get('jsonp', False)
cluster = queryParams.get('cluster', False)
def find_matches():
matches = []
for root, dirs, files in os.walk(settings.WHISPER_DIR):
root = root.replace(settings.WHISPER_DIR, '')
for basename in files:
if fnmatch.fnmatch(basename, '*.wsp'):
matches.append(os.path.join(root, basename))
for root, dirs, files in os.walk(settings.CERES_DIR):
root = root.replace(settings.CERES_DIR, '')
for filename in files:
if filename == '.ceres-node':
matches.append(root)
# unlike 0.9.x, we're going to use os.walk with followlinks
# since we require Python 2.7 and newer that supports it
if RRDReader.supported:
for root, dirs, files in os.walk(settings.RRD_DIR, followlinks=True):
root = root.replace(settings.RRD_DIR, '')
for basename in files:
if fnmatch.fnmatch(basename, '*.rrd'):
absolute_path = os.path.join(settings.RRD_DIR, root, basename)
(basename,extension) = os.path.splitext(basename)
metric_path = os.path.join(root, basename)
rrd = RRDReader(absolute_path, metric_path)
for datasource_name in rrd.get_datasources(absolute_path):
matches.append(os.path.join(metric_path, datasource_name))
matches = [
m
.replace('.wsp', '')
.replace('.rrd', '')
.replace('/', '.')
.lstrip('.')
for m in sorted(matches)
]
return matches
matches = []
if cluster and len(settings.CLUSTER_SERVERS) >= 1:
try:
matches = reduce( lambda x, y: list(set(x + y)), \
[json.loads(urllib.urlopen('http://' + cluster_server + '/metrics/index.json').read()) \
for cluster_server in settings.CLUSTER_SERVERS])
except urllib.URLError:
log.exception()
return json_response_for(request, matches, jsonp=jsonp, status=500)
else:
matches = find_matches()
return json_response_for(request, matches, jsonp=jsonp)
def load_template(request, name, val):
try:
template = Template.objects.get(name=name)
except Template.DoesNotExist:
return json_response( dict(error="Template '%s' does not exist. " % name) )
state = json.loads(template.loadState(val))
state['name'] = '%s/%s' % (name, val)
return json_response( dict(state=state) )
def load_template(request, name, host_id):
try:
template = Template.objects.get(name=name)
except Template.DoesNotExist:
return json_response( dict(error="Template '%s' does not exist. " % name) )
state = json.loads(template.loadState(host_id))
state['name'] = '%s/%s' % (name, host_id)
return json_response( dict(state=state) )
def test_expand_view(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
url = reverse('metrics_expand')
# default
request = {'query': '*'}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(data['results'], [u'hosts'])
# empty query
request = {'query': ''}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(data['results'], [u''])
def test_dashboard_load_template_existing(self):
url = reverse('dashboard_save_template',
args=['testtemplate', 'testkey'])
request = copy.deepcopy(self.testtemplate)
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
url = reverse('dashboard_load_template',
args=['testtemplate', 'testkey'])
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
graph_data = json.loads(self.testtemplate["state"].replace(
'__VALUE__', 'testkey'))
self.assertEqual(
data,
json.loads(
'{"state": {"name": "testtemplate/testkey", "graphs": ' +
json.dumps(graph_data['graphs']) + '}}'))
def test_expand_view(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
url = reverse('metrics_expand')
# default
request = {'query': '*'}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(data['results'], [u'hosts'])
# empty query
request = {'query': ''}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(data['results'], [u''])
def test_index_json(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
url = reverse('metrics_index')
# default
request = {}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(data[0], 'hosts.worker1.cpu')
self.assertEqual(data[1], 'hosts.worker2.cpu')
# XXX Disabling this test for now since a local running
# Graphite webapp will always return a 200, breaking our test
## cluster failure
#request = {'cluster': 1}
#response = self.client.post(url, request)
#self.assertEqual(response.status_code, 500)
#data = json.loads(response.content)
#self.assertEqual(data, [])
# jsonp
request = {'jsonp': 'callback'}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content.split(b"(")[1].strip(b")"))
self.assertEqual(data[0], 'hosts.worker1.cpu')
self.assertEqual(data[1], 'hosts.worker2.cpu')
# failure
def mock_STORE_get_index(self, requestContext=None):
raise Exception('test')
with patch('graphite.metrics.views.STORE.get_index',
mock_STORE_get_index):
request = {}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 500)
data = json.loads(response.content)
self.assertEqual(data, [])
def test_get_detail_json_object_does_not_exist(self):
url = reverse('events_detail', args=[1])
response = self.client.get(url, {}, HTTP_ACCEPT='application/json')
self.assertEqual(response.status_code, 404)
event = json.loads(response.content)
self.assertEqual(event['error'], 'Event matching query does not exist')
url = reverse('events_detail', args=[1])
response = self.client.get(url, {})
self.assertEqual(response.status_code, 404)
self.assertRegexpMatches(response.content, b'<h1>Not Found</h1>')
def index_json(request):
jsonp = request.REQUEST.get('jsonp', False)
cluster = request.REQUEST.get('cluster', False)
if cluster and len(settings.CLUSTER_SERVERS) > 1:
matches = reduce( lambda x, y: list(set(x + y)), \
[json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read()) \
for cluster_server in settings.CLUSTER_SERVERS])
else:
matches = [ node for finder in settings.STORAGE_FINDERS for node in get_finder(finder).get_all_nodes() ]
return json_response_for(request, matches, jsonp=jsonp)
def test_index_json(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
url = reverse('metrics_index')
# default
request = {}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(data[0], 'hosts.worker1.cpu')
self.assertEqual(data[1], 'hosts.worker2.cpu')
# XXX Disabling this test for now since a local running
# Graphite webapp will always return a 200, breaking our test
## cluster failure
#request = {'cluster': 1}
#response = self.client.post(url, request)
#self.assertEqual(response.status_code, 500)
#data = json.loads(response.content)
#self.assertEqual(data, [])
# jsonp
request = {'jsonp': 'callback'}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content.split(b"(")[1].strip(b")"))
self.assertEqual(data[0], 'hosts.worker1.cpu')
self.assertEqual(data[1], 'hosts.worker2.cpu')
# failure
def mock_STORE_get_index(self, requestContext=None):
raise Exception('test')
with patch('graphite.metrics.views.STORE.get_index', mock_STORE_get_index):
request = {}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 500)
data = json.loads(response.content)
self.assertEqual(data, [])
def parseDataOptions(data):
queryParams = MultiValueDict()
try:
options = json.loads(data)
for k, v in options.items():
if isinstance(v, list):
queryParams.setlist(k, v)
else:
queryParams[k] = unicode(v)
except:
log.exception('json_request decode error')
return parseOptionsDictionary(queryParams)
def test_unicode_graph_name(self):
url = reverse('browser_my_graph')
user = User.objects.create_user('test', '*****@*****.**', 'pass')
self.client.login(username='******', password='******')
response = self.client.get(url, {'path': ''})
self.assertEqual(response.status_code, 200)
user.profile.mygraph_set.create(name=u'fòo', url='bar')
response = self.client.get(url, {'path': ''})
self.assertEqual(response.status_code, 200)
[leaf] = json.loads(response.content)
self.assertEqual(leaf['text'], u'fòo')
def test_unicode_graph_name(self):
url = reverse('browser_my_graph')
user = User.objects.create_user('test', '*****@*****.**', 'pass')
self.client.login(username='******', password='******')
response = self.client.get(url, {'path': ''})
self.assertEqual(response.status_code, 200)
user.profile.mygraph_set.create(name=u'fòo', url='bar')
response = self.client.get(url, {'path': ''})
self.assertEqual(response.status_code, 200)
[leaf] = json.loads(response.content)
self.assertEqual(leaf['text'], u'fòo')
def parseDataOptions(data):
queryParams = MultiValueDict()
try:
options = json.loads(data)
for k,v in options.items():
if isinstance(v, list):
queryParams.setlist(k, v)
else:
queryParams[k] = unicode(v)
except:
log.exception('json_request decode error')
return parseOptionsDictionary(queryParams)
def save(request, name):
# Deserialize and reserialize as a validation step
state = str( json.dumps( json.loads( request.POST['state'] ) ) )
try:
dashboard = Dashboard.objects.get(name=name)
except Dashboard.DoesNotExist:
dashboard = Dashboard.objects.create(name=name, state=state)
else:
dashboard.state = state
dashboard.save();
return json_response( dict(success=True) )
def test_get_metadata_view(self):
"""Stub to test get_metadata_view. This currently doesn't test a valid key """
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
url = reverse('metrics_get_metadata')
# bad key
request = {'metric': 'hosts.worker1.cpu', 'key': 'a'}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(data['hosts.worker1.cpu']['error'], "Unexpected error occurred in CarbonLink.get_metadata(hosts.worker1.cpu, a)")
def save(request, name):
# Deserialize and reserialize as a validation step
state = str( json.dumps( json.loads( request.POST['state'] ) ) )
try:
dashboard = Dashboard.objects.get(name=name)
except Dashboard.DoesNotExist:
dashboard = Dashboard.objects.create(name=name, state=state)
else:
dashboard.state = state
dashboard.save();
return json_response( dict(success=True) )
def test_set_metadata_view(self):
"""Stub to test set_metadata_view. This currently doesn't test a valid key """
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
url = reverse('metrics_set_metadata')
# GET
# bad key
request = {'metric': 'hosts.worker1.cpu', 'key': 'a', 'value': 'b'}
response = self.client.get(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(data['hosts.worker1.cpu']['error'], "Unexpected error occurred in CarbonLink.set_metadata(hosts.worker1.cpu, a)")
# POST
# bad key
request = {'operations': '[{ "metric": "hosts.worker1.cpu", "key": "a", "value": "b" }]'}
response = self.client.post(url, request)
self.assertEqual(response.status_code, 200)
data = json.loads(response.content)
self.assertEqual(data['hosts.worker1.cpu']['error'], "Unexpected error occurred in bulk CarbonLink.set_metadata(hosts.worker1.cpu)")
def create_temporary(request):
state = str( json.dumps( json.loads( request.POST['state'] ) ) )
i = 0
while True:
name = "temporary-%d" % i
try:
Dashboard.objects.get(name=name)
except Dashboard.DoesNotExist:
dashboard = Dashboard.objects.create(name=name, state=state)
break
else:
i += 1
return json_response( dict(name=dashboard.name) )
def create_temporary(request):
state = str(json.dumps(json.loads(request.POST['state'])))
i = 0
while True:
name = "temporary-%d" % i
try:
Dashboard.objects.get(name=name)
except Dashboard.DoesNotExist:
dashboard = Dashboard.objects.create(name=name, state=state)
break
else:
i += 1
return json_response(dict(name=dashboard.name))
def save_template(request, name):
# Deserialize and reserialize as a validation step
state = str( json.dumps( json.loads( request.POST['state'] ) ) )
try:
template = Template.objects.get(name=name)
except Template.DoesNotExist:
template = Template.objects.create(name=name)
template.setState(state)
template.save()
else:
template.setState(state)
template.save();
return json_response( dict(success=True) )
def save(request, name):
if 'change' not in getPermissions(request.user):
return json_response( dict(error="Must be logged in with appropriate permissions to save") )
# Deserialize and reserialize as a validation step
state = str( json.dumps( json.loads( request.POST['state'] ) ) )
try:
dashboard = Dashboard.objects.get(name=name)
except Dashboard.DoesNotExist:
dashboard = Dashboard.objects.create(name=name, state=state)
else:
dashboard.state = state
dashboard.save();
return json_response( dict(success=True) )
def template(request, name, val):
template_conf_missing = False
try:
config.check()
except OSError as e:
if e.errno == errno.ENOENT:
template_conf_missing = True
else:
raise
initialError = None
debug = request.GET.get('debug', False)
theme = request.GET.get('theme', config.ui_config['theme'])
css_file = finders.find('css/dashboard-%s.css' % theme)
if css_file is None:
initialError = "Invalid theme '%s'" % theme
theme = config.ui_config['theme']
context = {
'schemes_json' : json.dumps(config.schemes),
'ui_config_json' : json.dumps(config.ui_config),
'jsdebug' : debug or settings.JAVASCRIPT_DEBUG,
'debug' : debug,
'theme' : theme,
'initialError' : initialError,
'querystring' : json.dumps( dict( request.GET.items() ) ),
'template_conf_missing' : template_conf_missing,
'userName': '',
'permissions': json.dumps(getPermissions(request.user)),
'permissionsUnauthenticated': json.dumps(getPermissions(None))
}
user = request.user
if user:
context['userName'] = user.username
try:
template = Template.objects.get(name=name)
except Template.DoesNotExist:
context['initialError'] = "Template '%s' does not exist." % name
else:
state = json.loads(template.loadState(val))
state['name'] = '%s/%s' % (name, val)
context['initialState'] = json.dumps(state)
return render_to_response("dashboard.html", context)
def save_template(request, name, key):
if 'change' not in getPermissions(request.user):
return json_response( dict(error="Must be logged in with appropriate permissions to save the template") )
# Deserialize and reserialize as a validation step
state = str( json.dumps( json.loads( request.POST['state'] ) ) )
try:
template = Template.objects.get(name=name)
except Template.DoesNotExist:
template = Template.objects.create(name=name)
template.setState(state)
template.save()
else:
template.setState(state, key)
template.save();
return json_response( dict(success=True) )
def setState(self, state):
def replace_hostid(s):
if isinstance(s, unicode):
s = self.__class__.metric_path_re.sub('__HOST_ID__.', s)
return s
def update_graph(graph):
graph_opts = graph[1]
graph_opts['target'] = [replace_hostid(s) for s in graph_opts['target']]
return [replace_hostid(graph[0]),
graph_opts,
replace_hostid(graph[2])]
# Parse JSON here and replace first five elements of target with __HOST_ID__
parsed_state = json.loads(state)
for i, graph in enumerate(parsed_state['graphs']):
parsed_state['graphs'][i] = update_graph(graph)
self.state = json.dumps(parsed_state)
