def set_metadata_view(request): results = {} if request.method == "GET": metric = request.GET["metric"] key = request.GET["key"] value = request.GET["value"] try: results[metric] = CarbonLink.set_metadata(metric, key, value) except: log.exception() results[metric] = dict(error="Unexpected error occurred in CarbonLink.set_metadata(%s, %s)" % (metric, key)) elif request.method == "POST": if request.META.get("CONTENT_TYPE") == "application/json": operations = json.loads(request.body) else: operations = json.loads(request.POST["operations"]) for op in operations: metric = None try: metric, key, value = op["metric"], op["key"], op["value"] results[metric] = CarbonLink.set_metadata(metric, key, value) except: log.exception() if metric: results[metric] = dict( error="Unexpected error occurred in bulk CarbonLink.set_metadata(%s)" % metric ) else: results = dict(error="Invalid request method") return json_response_for(request, results)
def test_set_metadata_view(self): """Stub to test set_metadata_view. This currently doesn't test a valid key """ self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) url = reverse('metrics_set_metadata') # GET # bad key request = {'metric': 'hosts.worker1.cpu', 'key': 'a', 'value': 'b'} response = self.client.get(url, request) self.assertEqual(response.status_code, 200) data = json.loads(response.content) self.assertEqual( data['hosts.worker1.cpu']['error'], "Unexpected error occurred in CarbonLink.set_metadata(hosts.worker1.cpu, a)" ) # POST # bad key request = { 'operations': '[{ "metric": "hosts.worker1.cpu", "key": "a", "value": "b" }]' } response = self.client.post(url, request) self.assertEqual(response.status_code, 200) data = json.loads(response.content) self.assertEqual( data['hosts.worker1.cpu']['error'], "Unexpected error occurred in bulk CarbonLink.set_metadata(hosts.worker1.cpu)" )
def request(self, method, url, fields, requestContext=None): headers = requestContext.get('forwardHeaders') if requestContext else {} if 'Authorization' not in headers and self.username and self.password: user_pw = '%s:%s' % (self.username, self.password) if sys.version_info[0] >= 3: user_pw_b64 = b2a_base64(user_pw.encode('utf-8')).decode('ascii') else: user_pw_b64 = user_pw.encode('base64') headers['Authorization'] = 'Basic ' + user_pw_b64 req_fields = [] for (field, value) in fields.items(): if value is None: continue if isinstance(value, list) or isinstance(value, tuple): req_fields.extend([(field, v) for v in value if v is not None]) else: req_fields.append((field, value)) result = http.request( method, self.base_url + url, fields=req_fields, headers=headers, timeout=self.settings.FIND_TIMEOUT, ) if result.status == 400: raise ValueError(json.loads(result.data.decode('utf-8')).get('error')) if result.status != 200: raise Exception('HTTP Error from remote tagdb: %s %s' % (result.status, result.data)) return json.loads(result.data.decode('utf-8'))
def request(self, method, url, fields, requestContext=None): headers = requestContext.get('forwardHeaders') if requestContext else {} if 'Authorization' not in headers and self.username and self.password: user_pw = '%s:%s' % (self.username, self.password) if sys.version_info[0] >= 3: user_pw_b64 = b2a_base64(user_pw.encode('utf-8')).decode('ascii') else: user_pw_b64 = user_pw.encode('base64') headers['Authorization'] = 'Basic ' + user_pw_b64 req_fields = [] for (field, value) in fields.items(): if value is None: continue if isinstance(value, list) or isinstance(value, tuple): req_fields.extend([(field, v) for v in value if v is not None]) else: req_fields.append((field, value)) result = http.request( method, self.base_url + url, fields=req_fields, headers=headers, timeout=self.settings.REMOTE_FIND_TIMEOUT, ) if result.status == 400: raise ValueError(json.loads(result.data.decode('utf-8')).get('error')) if result.status != 200: raise Exception('HTTP Error from remote tagdb: %s %s' % (result.status, result.data)) return json.loads(result.data.decode('utf-8'))
def test_timezone_handling(self): url = reverse('events') data = {'what': 'something happened', 'when': time.time() - 3590} with self.settings(TIME_ZONE='Europe/Moscow'): response = self.client.post(url, json.dumps(data), content_type='application/json') self.assertEqual(response.status_code, 200) event = Event.objects.get() self.assertTrue(timezone.now() - event.when < timedelta(hours=1)) self.assertTrue(timezone.now() - event.when > timedelta(seconds=3580)) url = reverse('events_get_data') with self.settings(TIME_ZONE='Europe/Berlin'): response1 = self.client.get(url) [event] = json.loads(response1.content) self.assertEqual(event['what'], 'something happened') with self.settings(TIME_ZONE='UTC'): response2 = self.client.get(url) self.assertEqual(response1.content, response2.content) url = reverse('events') data = {'what': 'something else happened'} with self.settings(TIME_ZONE='Asia/Hong_Kong'): response = self.client.post(url, json.dumps(data), content_type='application/json') self.assertEqual(response.status_code, 200) url = reverse('events_get_data') with self.settings(TIME_ZONE='Europe/Berlin'): response = self.client.get(url, {'from': int(time.time() - 3500)}) [event] = json.loads(response.content) self.assertEqual(event['what'], 'something else happened')
def test_get_detail_json_html(self): creation_url = reverse('events') event = { 'what': 'Something happened', 'data': 'more info', 'tags': ['foo', 'bar', 'baz'], } response = self.client.post(creation_url, json.dumps(event), content_type='application/json') self.assertEqual(response.status_code, 200) url = reverse('events_get_data') # should match two events using old set logic response = self.client.get(url, {'tags': 'foo bar baz'}) self.assertEqual(response.status_code, 200) events = json.loads(response.content) self.assertEqual(len(events), 1) url = reverse('events_detail', args=[events[0]['id']]) response = self.client.get(url, {}, HTTP_ACCEPT='application/json') self.assertEqual(response.status_code, 200) event = json.loads(response.content) self.assertEqual(event['what'], 'Something happened') self.assertEqual(event['tags'], ['foo', 'bar', 'baz']) url = reverse('events_detail', args=[events[0]['id']]) response = self.client.get(url, {}) self.assertEqual(response.status_code, 200) self.assertRegexpMatches( response.content, re.compile( b'^<html>.+<title>Something happened</title>.+<h1>Something happened</h1>.+<td>tags</td><td>foo bar baz</td>.+</html>$', re.DOTALL))
def set_metadata_view(request): results = {} if request.method == 'GET': metric = request.GET['metric'] key = request.GET['key'] value = request.GET['value'] try: results[metric] = CarbonLink.set_metadata(metric, key, value) except: log.exception() results[metric] = dict(error="Unexpected error occurred in CarbonLink.set_metadata(%s, %s)" % (metric, key)) elif request.method == 'POST': if request.META.get('CONTENT_TYPE') == 'application/json': operations = json.loads( request.body ) else: operations = json.loads( request.POST['operations'] ) for op in operations: metric = None try: metric, key, value = op['metric'], op['key'], op['value'] results[metric] = CarbonLink.set_metadata(metric, key, value) except: log.exception() if metric: results[metric] = dict(error="Unexpected error occurred in bulk CarbonLink.set_metadata(%s)" % metric) else: results = dict(error="Invalid request method") return json_response_for(request, results)
def test_event_tags(self): url = reverse('events_get_data') response = self.client.get(url) self.assertEqual(response.status_code, 200) self.assertEqual(len(json.loads(response.content)), 0) creation_url = reverse('events') event = { 'what': 'Something happened', 'data': 'more info', 'tags': ['foo', 'bar'], } response = self.client.post(creation_url, json.dumps(event), content_type='application/json') self.assertEqual(response.status_code, 200) event = Event.objects.get() self.assertEqual(event.what, 'Something happened') self.assertEqual(event.tags, 'foo bar') response = self.client.get(url) self.assertEqual(response.status_code, 200) events = json.loads(response.content) self.assertEqual(len(events), 1) event = events[0] self.assertEqual(event['what'], 'Something happened') self.assertEqual(event['tags'], ['foo', 'bar']) response = self.client.get(creation_url) self.assertEqual(response.status_code, 200) expected_re = b'^<html>.+<title>Events</title>.+Something happened.+<td>\\['foo', 'bar'\\]</td>.+</html>$' self.assertRegexpMatches(response.content, re.compile(expected_re, re.DOTALL))
def test_dashboard_login_invalid_authenticate(self): url = reverse('dashboard_login') request = {"username": "******", "password": "******"} response = self.client.post(url, request) self.assertEqual(response.status_code, 200) self.assertEqual(json.loads(response.content), json.loads('{"errors": {"reason": "Username and/or password invalid."}, "success": false, "text": {}, "permissions": []}'))
def test_dashboard_logout(self): url = reverse('dashboard_logout') request = {"username": "******"} response = self.client.post(url, request) self.assertEqual(response.status_code, 200) self.assertEqual( json.loads(response.content), json.loads('{"errors": {}, "success": true, "text": {}}'))
def test_tag_views(self): url = reverse('tagList') expected = 'test.a;blah=blah;hello=tiger' response = self.client.post(url + '/tagSeries', {'path': 'test.a;hello=tiger;blah=blah'}) self.assertEqual(response['Content-Type'], 'application/json') self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True)) expected = [{"tag": "hello"}] response = self.client.get(url, {'filter': 'hello$'}) self.assertEqual(response['Content-Type'], 'application/json') result = json.loads(response.content) self.assertEqual(len(result), len(expected)) self.assertEqual(result[0]['tag'], expected[0]['tag']) response = self.client.get(url, {'filter': 'hello$', 'pretty': 1}) self.assertEqual(response['Content-Type'], 'application/json') result = json.loads(response.content) self.assertEqual(len(result), len(expected)) self.assertEqual(result[0]['tag'], expected[0]['tag']) expected = {"tag": "hello", "values": [{"count": 1, "value": "tiger"}]} response = self.client.get(url + '/hello', {'filter': 'tiger$'}) self.assertEqual(response['Content-Type'], 'application/json') result = json.loads(response.content) self.assertEqual(result['tag'], expected['tag']) self.assertEqual(len(result['values']), len(expected['values'])) self.assertEqual(result['values'][0]['count'], expected['values'][0]['count']) self.assertEqual(result['values'][0]['value'], expected['values'][0]['value']) response = self.client.get(url + '/hello', {'filter': 'tiger$', 'pretty': 1}) self.assertEqual(response['Content-Type'], 'application/json') result = json.loads(response.content) self.assertEqual(result['tag'], expected['tag']) self.assertEqual(len(result['values']), len(expected['values'])) self.assertEqual(result['values'][0]['count'], expected['values'][0]['count']) self.assertEqual(result['values'][0]['value'], expected['values'][0]['value']) expected = ['test.a;blah=blah;hello=tiger'] response = self.client.get(url + '/findSeries?expr[]=name=test.a&expr[]=hello=tiger&expr[]=blah=blah&pretty=1') self.assertEqual(response['Content-Type'], 'application/json') self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True)) expected = True response = self.client.post(url + '/delSeries', {'path': 'test.a;blah=blah;hello=tiger'}) self.assertEqual(response['Content-Type'], 'application/json') self.assertEqual(response.content, json.dumps(expected)) expected = [] response = self.client.get(url + '/findSeries?expr=name=test.a&expr=hello=tiger&expr=blah=blah') self.assertEqual(response['Content-Type'], 'application/json') self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True))
def test_dashboard_login_valid_authenticate(self, authenticate): url = reverse('dashboard_login') request = {"username": "******", "password": "******"} user = User.objects.create(email='*****@*****.**') user.backend = '' authenticate.return_value = user response = self.client.post(url, request) self.assertEqual(response.status_code, 200) self.assertEqual(json.loads(response.content), json.loads('{"permissions": ["change", "delete"], "success": true, "text": {}, "errors": {}}'))
def test_dashboard_login_invalid_authenticate(self): url = reverse('dashboard_login') request = {"username": "******", "password": "******"} response = self.client.post(url, request) self.assertEqual(response.status_code, 200) self.assertEqual( json.loads(response.content), json.loads( '{"errors": {"reason": "Username and/or password invalid."}, "success": false, "text": {}, "permissions": []}' ))
def test_dashboard_login_valid_authenticate_not_active(self, authenticate): url = reverse('dashboard_login') request = {"username": "******", "password": "******"} user = User.objects.create(email='*****@*****.**') user.backend = '' user.is_active = False authenticate.return_value = user response = self.client.post(url, request) self.assertEqual(response.status_code, 200) self.assertEqual(json.loads(response.content), json.loads('{"permissions": [], "success": false, "errors": {"reason": "Account disabled."}, "text": {}}'))
def test_dashboard_load_template_existing(self): url = reverse('dashboard_save_template', args=['testtemplate', 'testkey']) request = copy.deepcopy(self.testtemplate) response = self.client.post(url, request) self.assertEqual(response.status_code, 200) url = reverse('dashboard_load_template', args=['testtemplate', 'testkey']) response = self.client.get(url) self.assertEqual(response.status_code, 200) data = json.loads(response.content) graph_data = json.loads(self.testtemplate["state"].replace('__VALUE__', 'testkey')) self.assertEqual(data, json.loads('{"state": {"name": "testtemplate/testkey", "graphs": ' + json.dumps(graph_data['graphs']) + '}}'))
def test_tag_sets(self): creation_url = reverse('events') events = [ { 'what': 'Something happened', 'data': 'more info', 'tags': ['foo'], }, { 'what': 'Something else happened', 'data': 'even more info', 'tags': ['bar'], }, { 'what': 'A final thing happened', 'data': 'yet even more info', 'tags': ['foo', 'bar'], }, ] for event in events: response = self.client.post(creation_url, json.dumps(event), content_type='application/json') self.assertEqual(response.status_code, 200) url = reverse('events_get_data') # should match two events using old set logic response = self.client.get(url, {'tags': 'foo'}) self.assertEqual(response.status_code, 200) events = json.loads(response.content) self.assertEqual(len(events), 2) # should match one event using set intersection response = self.client.get("%s%s" % (url, '?tags=foo%20bar&set=intersection')) self.assertEqual(response.status_code, 200) events = json.loads(response.content) self.assertEqual(len(events), 1) # should match zero events using set intersection and unknown tag response = self.client.get( "%s%s" % (url, '?tags=foo%20bar%20nope&set=intersection')) self.assertEqual(response.status_code, 200) events = json.loads(response.content) self.assertEqual(len(events), 0) # should match all events using set union response = self.client.get("%s%s" % (url, '?tags=foo%20bar&set=union')) self.assertEqual(response.status_code, 200) events = json.loads(response.content) self.assertEqual(len(events), 3)
def test_dashboard_login_valid_authenticate(self, authenticate): url = reverse('dashboard_login') request = {"username": "******", "password": "******"} user = User.objects.create(email='*****@*****.**') user.backend = '' authenticate.return_value = user response = self.client.post(url, request) self.assertEqual(response.status_code, 200) self.assertEqual( json.loads(response.content), json.loads( '{"permissions": ["change", "delete"], "success": true, "text": {}, "errors": {}}' ))
def test_dashboard_login_valid_authenticate_not_active(self, authenticate): url = reverse('dashboard_login') request = {"username": "******", "password": "******"} user = User.objects.create(email='*****@*****.**') user.backend = '' user.is_active = False authenticate.return_value = user response = self.client.post(url, request) self.assertEqual(response.status_code, 200) self.assertEqual( json.loads(response.content), json.loads( '{"permissions": [], "success": false, "errors": {"reason": "Account disabled."}, "text": {}}' ))
def post_event(request): if request.method == 'POST': event = json.loads(request.body) assert isinstance(event, dict) tags = event.get('tags') if tags: if not isinstance(tags, list): return HttpResponse( json.dumps({'error': '"tags" must be an array'}), status=400) tags = ' '.join(tags) if 'when' in event: when = make_aware( datetime.datetime.utcfromtimestamp( event.get('when')), pytz.utc) else: when = now() Event.objects.create( what=event.get('what'), tags=tags, when=when, data=event.get('data', ''), ) return HttpResponse(status=200) else: return HttpResponse(status=405)
def test_unicode_usergraph(self): url = reverse('browser_usergraph') user = User.objects.create_user('tèst', '*****@*****.**', 'pass') self.client.login(username='******', password='******') self.client.get( reverse('browser_header')) # this creates a profile for the user user.profile.mygraph_set.create(name=u'fòo', url='bar') response = self.client.get( url, { 'query': 'tèst.*', 'format': 'treejson', 'path': 'tèst', 'user': '******', 'node': 'tèst' }) self.assertEqual(response.status_code, 200) [leaf] = json.loads(response.content) self.assertEqual( leaf, { u'leaf': 1, u'text': u'fòo', u'allowChildren': 0, u'graphUrl': u'bar', u'id': u'tèst.845aa5781192007e1866648eea9f7355', u'expandable': 0, })
def post_event(request): if request.method == 'POST': event = json.loads(request.body) assert isinstance(event, dict) tags = event.get('tags') if tags is not None: if isinstance(tags, list): tags = ' '.join(tags) elif not isinstance(tags, six.string_types): return HttpResponse( json.dumps({'error': '"tags" must be an array or space-separated string'}), status=400) else: tags = None if 'when' in event: when = epoch_to_dt(event['when']) else: when = now() Event.objects.create( what=event.get('what'), tags=tags, when=when, data=event.get('data', ''), ) return HttpResponse(status=200) else: return HttpResponse(status=405)
def index_all_json(request): if len(settings.CLUSTER_SERVERS) <= 1: return index_json(request) matches = [] for cluster_server in settings.CLUSTER_SERVERS: matches = list(set(matches + json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read()))) return HttpResponse(json.dumps(matches), mimetype='application/json')
def post_event(request): if request.method == 'POST': try: event = json.loads(request.body) except ValueError: # try to see if this is a heroku post hook if "app" in request.POST and "user" in request.POST and "head" in request.POST: event = { "what":"%(app)s release" % request.POST, "tags":"heroku,%(app)s" % request.POST, "data":"%(app)s released to %(head)s by %(user)s" % request.POST } assert isinstance(event, dict) values = {} values["what"] = event["what"] values["tags"] = event.get("tags", None) values["when"] = datetime.datetime.fromtimestamp( event.get("when", time.time())) if "data" in event: values["data"] = event["data"] e = models.Event(**values) e.save() return HttpResponse(status=200) else: return HttpResponse(status=405)
def test_tag_as_str(self): creation_url = reverse('events') event = { 'what': 'Something happened', 'data': 'more info', 'tags': 'other', } response = self.client.post(creation_url, json.dumps(event), content_type='application/json') self.assertEqual(response.status_code, 200) event = Event.objects.get() self.assertEqual(event.what, 'Something happened') self.assertEqual(event.tags, 'other') url = reverse('events_get_data') response = self.client.get(url, {'until': 'now+5min', 'jsonp': 'test'}) self.assertEqual(response.status_code, 200) self.assertRegexpMatches(response.content, b'^test[(].+[)]$') events = json.loads(response.content[5:-1]) self.assertEqual(len(events), 1) event = events[0] self.assertEqual(event['what'], 'Something happened') self.assertEqual(event['tags'], ['other'])
def index_json(request): jsonp = request.REQUEST.get("jsonp", False) cluster = request.REQUEST.get("cluster", False) def find_matches(): matches = [] for root, dirs, files in os.walk(settings.WHISPER_DIR): root = root.replace(settings.WHISPER_DIR, "") for basename in files: if fnmatch.fnmatch(basename, "*.wsp"): matches.append(os.path.join(root, basename)) for root, dirs, files in os.walk(settings.CERES_DIR): root = root.replace(settings.CERES_DIR, "") for filename in files: if filename == ".ceres-node": matches.append(root) matches = [m.replace(".wsp", "").replace(".rrd", "").replace("/", ".").lstrip(".") for m in sorted(matches)] return matches matches = [] if cluster and len(settings.CLUSTER_SERVERS) > 1: matches = reduce( lambda x, y: list(set(x + y)), [ json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read()) for cluster_server in settings.CLUSTER_SERVERS ], ) else: matches = find_matches() return json_response_for(request, matches, jsonp=jsonp)
def post_event(request): if request.method == 'POST': event = json.loads(request.body) assert isinstance(event, dict) tags = event.get('tags') if tags: if not isinstance(tags, list): return HttpResponse(json.dumps( {'error': '"tags" must be an array'}), status=400) tags = ' '.join(tags) if 'when' in event: when = make_aware( datetime.datetime.utcfromtimestamp(event.get('when')), pytz.utc) else: when = now() Event.objects.create( what=event.get('what'), tags=tags, when=when, data=event.get('data', ''), ) return HttpResponse(status=200) else: return HttpResponse(status=405)
def load(request, name): try: dashboard = Dashboard.objects.get(name=name) except Dashboard.DoesNotExist: return json_response( dict(error="Dashboard '%s' does not exist. " % name) ) return json_response( dict(state=json.loads(dashboard.state)) )
def index_json(request): jsonp = request.REQUEST.get('jsonp', False) cluster = request.REQUEST.get('cluster', False) def find_matches(): matches = [] for root, dirs, files in os.walk(settings.WHISPER_DIR): root = root.replace(settings.WHISPER_DIR, '') for basename in files: if fnmatch.fnmatch(basename, '*.wsp'): matches.append(os.path.join(root, basename)) for root, dirs, files in os.walk(settings.CERES_DIR): root = root.replace(settings.CERES_DIR, '') for filename in files: if filename == '.ceres-node': matches.append(root) matches = [ m.replace('.wsp', '').replace('.rrd', '').replace('/', '.').lstrip('.') for m in sorted(matches) ] return matches matches = [] if cluster and len(settings.CLUSTER_SERVERS) > 1: matches = reduce( lambda x, y: list(set(x + y)), \ [json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read()) \ for cluster_server in settings.CLUSTER_SERVERS]) else: matches = find_matches() return json_response_for(request, matches, jsonp=jsonp)
def post_event(request): if request.method == 'POST': event = json.loads(request.body) assert isinstance(event, dict) tags = event.get('tags') if tags is not None: if isinstance(tags, list): tags = ' '.join(tags) elif not isinstance(tags, six.string_types): return HttpResponse(json.dumps({ 'error': '"tags" must be an array or space-separated string' }), status=400) else: tags = None if 'when' in event: when = epoch_to_dt(event['when']) else: when = now() Event.objects.create( what=event.get('what'), tags=tags, when=when, data=event.get('data', ''), ) return HttpResponse(status=200) else: return HttpResponse(status=405)
def index_json(request): queryParams = request.GET.copy() queryParams.update(request.POST) jsonp = queryParams.get('jsonp', False) cluster = queryParams.get('cluster', False) def find_matches(): matches = [] for root, dirs, files in os.walk(settings.WHISPER_DIR): root = root.replace(settings.WHISPER_DIR, '') for basename in files: if fnmatch.fnmatch(basename, '*.wsp'): matches.append(os.path.join(root, basename)) for root, dirs, files in os.walk(settings.CERES_DIR): root = root.replace(settings.CERES_DIR, '') for filename in files: if filename == '.ceres-node': matches.append(root) # unlike 0.9.x, we're going to use os.walk with followlinks # since we require Python 2.7 and newer that supports it if RRDReader.supported: for root, dirs, files in os.walk(settings.RRD_DIR, followlinks=True): root = root.replace(settings.RRD_DIR, '') for basename in files: if fnmatch.fnmatch(basename, '*.rrd'): absolute_path = os.path.join(settings.RRD_DIR, root, basename) (basename, extension) = os.path.splitext(basename) metric_path = os.path.join(root, basename) rrd = RRDReader(absolute_path, metric_path) for datasource_name in rrd.get_datasources( absolute_path): matches.append( os.path.join(metric_path, datasource_name)) matches = [ m.replace('.wsp', '').replace('.rrd', '').replace('/', '.').lstrip('.') for m in sorted(matches) ] return matches matches = [] if cluster and len(settings.CLUSTER_SERVERS) >= 1: try: matches = reduce( lambda x, y: list(set(x + y)), \ [json.loads(urllib.urlopen('http://' + cluster_server + '/metrics/index.json').read()) \ for cluster_server in settings.CLUSTER_SERVERS]) except urllib.URLError: log.exception() return json_response_for(request, matches, jsonp=jsonp, status=500) else: matches = find_matches() return json_response_for(request, matches, jsonp=jsonp)
def full_path_zon_test(request): # 0. Parse parameters queryParams = request.GET.copy() queryParams.update(request.POST) # TODO: remove the default values, the default values are what we are using currently port = queryParams.get('port', '31509') metric_name = queryParams.get('metric', 'test.fullstack.graphite') # 1. Send data to graphite random_data = _send_random_data(metric_name) # 2. Wait (allow some latency) time.sleep(1) # 3. Query graphite try: res = urlopen( "http://localhost:{0}/render/?format=json&target={1}&from=-1min&noCache" .format(port, metric_name)) s = res.read().decode('utf-8') json_obj = json.loads(s) except Exception: raise # 4. Check Result result = "" details = "Expected value: {0}".format(random_data) if not json_obj: result = "fail" details = "graphite query response is empty" else: json_obj = json_obj[0] if "target" not in json_obj: result = "fail" details = "target field is missing" elif "datapoints" not in json_obj: result = "fail" details = "datapoints field is missing" elif len(json_obj["datapoints"]) != 1: result = "fail" details = "the number of returned datapoints is inconsistent" elif json_obj["datapoints"][0][0] is None: result = "fail" details = "datapoint is returned but its value is None" else: real_value = int(json_obj["datapoints"][0][0]) expected_value = random_data if real_value == expected_value: result = "pass" else: result = "fail" details = "Expected value: {0}, Real Value: {1}".format( expected_value, real_value) # 5. Response result_json_obj = {"result": result, "details": details} response = HttpResponse(content=json.dumps(result_json_obj), content_type='application/json') return response
def index_json(request): queryParams = request.GET.copy() queryParams.update(request.POST) jsonp = queryParams.get('jsonp', False) cluster = queryParams.get('cluster', False) def find_matches(): matches = [] for root, dirs, files in os.walk(settings.WHISPER_DIR): root = root.replace(settings.WHISPER_DIR, '') for basename in files: if fnmatch.fnmatch(basename, '*.wsp'): matches.append(os.path.join(root, basename)) for root, dirs, files in os.walk(settings.CERES_DIR): root = root.replace(settings.CERES_DIR, '') for filename in files: if filename == '.ceres-node': matches.append(root) # unlike 0.9.x, we're going to use os.walk with followlinks # since we require Python 2.7 and newer that supports it if RRDReader.supported: for root, dirs, files in os.walk(settings.RRD_DIR, followlinks=True): root = root.replace(settings.RRD_DIR, '') for basename in files: if fnmatch.fnmatch(basename, '*.rrd'): absolute_path = os.path.join(settings.RRD_DIR, root, basename) (basename,extension) = os.path.splitext(basename) metric_path = os.path.join(root, basename) rrd = RRDReader(absolute_path, metric_path) for datasource_name in rrd.get_datasources(absolute_path): matches.append(os.path.join(metric_path, datasource_name)) matches = [ m .replace('.wsp', '') .replace('.rrd', '') .replace('/', '.') .lstrip('.') for m in sorted(matches) ] return matches matches = [] if cluster and len(settings.CLUSTER_SERVERS) >= 1: try: matches = reduce( lambda x, y: list(set(x + y)), \ [json.loads(urllib.urlopen('http://' + cluster_server + '/metrics/index.json').read()) \ for cluster_server in settings.CLUSTER_SERVERS]) except urllib.URLError: log.exception() return json_response_for(request, matches, jsonp=jsonp, status=500) else: matches = find_matches() return json_response_for(request, matches, jsonp=jsonp)
def load_template(request, name, val): try: template = Template.objects.get(name=name) except Template.DoesNotExist: return json_response( dict(error="Template '%s' does not exist. " % name) ) state = json.loads(template.loadState(val)) state['name'] = '%s/%s' % (name, val) return json_response( dict(state=state) )
def load_template(request, name, host_id): try: template = Template.objects.get(name=name) except Template.DoesNotExist: return json_response( dict(error="Template '%s' does not exist. " % name) ) state = json.loads(template.loadState(host_id)) state['name'] = '%s/%s' % (name, host_id) return json_response( dict(state=state) )
def test_expand_view(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) url = reverse('metrics_expand') # default request = {'query': '*'} response = self.client.post(url, request) self.assertEqual(response.status_code, 200) data = json.loads(response.content) self.assertEqual(data['results'], [u'hosts']) # empty query request = {'query': ''} response = self.client.post(url, request) self.assertEqual(response.status_code, 200) data = json.loads(response.content) self.assertEqual(data['results'], [u''])
def test_dashboard_load_template_existing(self): url = reverse('dashboard_save_template', args=['testtemplate', 'testkey']) request = copy.deepcopy(self.testtemplate) response = self.client.post(url, request) self.assertEqual(response.status_code, 200) url = reverse('dashboard_load_template', args=['testtemplate', 'testkey']) response = self.client.get(url) self.assertEqual(response.status_code, 200) data = json.loads(response.content) graph_data = json.loads(self.testtemplate["state"].replace( '__VALUE__', 'testkey')) self.assertEqual( data, json.loads( '{"state": {"name": "testtemplate/testkey", "graphs": ' + json.dumps(graph_data['graphs']) + '}}'))
def test_index_json(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) url = reverse('metrics_index') # default request = {} response = self.client.post(url, request) self.assertEqual(response.status_code, 200) data = json.loads(response.content) self.assertEqual(data[0], 'hosts.worker1.cpu') self.assertEqual(data[1], 'hosts.worker2.cpu') # XXX Disabling this test for now since a local running # Graphite webapp will always return a 200, breaking our test ## cluster failure #request = {'cluster': 1} #response = self.client.post(url, request) #self.assertEqual(response.status_code, 500) #data = json.loads(response.content) #self.assertEqual(data, []) # jsonp request = {'jsonp': 'callback'} response = self.client.post(url, request) self.assertEqual(response.status_code, 200) data = json.loads(response.content.split(b"(")[1].strip(b")")) self.assertEqual(data[0], 'hosts.worker1.cpu') self.assertEqual(data[1], 'hosts.worker2.cpu') # failure def mock_STORE_get_index(self, requestContext=None): raise Exception('test') with patch('graphite.metrics.views.STORE.get_index', mock_STORE_get_index): request = {} response = self.client.post(url, request) self.assertEqual(response.status_code, 500) data = json.loads(response.content) self.assertEqual(data, [])
def test_get_detail_json_object_does_not_exist(self): url = reverse('events_detail', args=[1]) response = self.client.get(url, {}, HTTP_ACCEPT='application/json') self.assertEqual(response.status_code, 404) event = json.loads(response.content) self.assertEqual(event['error'], 'Event matching query does not exist') url = reverse('events_detail', args=[1]) response = self.client.get(url, {}) self.assertEqual(response.status_code, 404) self.assertRegexpMatches(response.content, b'<h1>Not Found</h1>')
def index_json(request): jsonp = request.REQUEST.get('jsonp', False) cluster = request.REQUEST.get('cluster', False) if cluster and len(settings.CLUSTER_SERVERS) > 1: matches = reduce( lambda x, y: list(set(x + y)), \ [json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read()) \ for cluster_server in settings.CLUSTER_SERVERS]) else: matches = [ node for finder in settings.STORAGE_FINDERS for node in get_finder(finder).get_all_nodes() ] return json_response_for(request, matches, jsonp=jsonp)
def parseDataOptions(data): queryParams = MultiValueDict() try: options = json.loads(data) for k, v in options.items(): if isinstance(v, list): queryParams.setlist(k, v) else: queryParams[k] = unicode(v) except: log.exception('json_request decode error') return parseOptionsDictionary(queryParams)
def test_unicode_graph_name(self): url = reverse('browser_my_graph') user = User.objects.create_user('test', '*****@*****.**', 'pass') self.client.login(username='******', password='******') response = self.client.get(url, {'path': ''}) self.assertEqual(response.status_code, 200) user.profile.mygraph_set.create(name=u'fòo', url='bar') response = self.client.get(url, {'path': ''}) self.assertEqual(response.status_code, 200) [leaf] = json.loads(response.content) self.assertEqual(leaf['text'], u'fòo')
def parseDataOptions(data): queryParams = MultiValueDict() try: options = json.loads(data) for k,v in options.items(): if isinstance(v, list): queryParams.setlist(k, v) else: queryParams[k] = unicode(v) except: log.exception('json_request decode error') return parseOptionsDictionary(queryParams)
def save(request, name): # Deserialize and reserialize as a validation step state = str( json.dumps( json.loads( request.POST['state'] ) ) ) try: dashboard = Dashboard.objects.get(name=name) except Dashboard.DoesNotExist: dashboard = Dashboard.objects.create(name=name, state=state) else: dashboard.state = state dashboard.save(); return json_response( dict(success=True) )
def test_get_metadata_view(self): """Stub to test get_metadata_view. This currently doesn't test a valid key """ self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) url = reverse('metrics_get_metadata') # bad key request = {'metric': 'hosts.worker1.cpu', 'key': 'a'} response = self.client.post(url, request) self.assertEqual(response.status_code, 200) data = json.loads(response.content) self.assertEqual(data['hosts.worker1.cpu']['error'], "Unexpected error occurred in CarbonLink.get_metadata(hosts.worker1.cpu, a)")
def test_set_metadata_view(self): """Stub to test set_metadata_view. This currently doesn't test a valid key """ self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) url = reverse('metrics_set_metadata') # GET # bad key request = {'metric': 'hosts.worker1.cpu', 'key': 'a', 'value': 'b'} response = self.client.get(url, request) self.assertEqual(response.status_code, 200) data = json.loads(response.content) self.assertEqual(data['hosts.worker1.cpu']['error'], "Unexpected error occurred in CarbonLink.set_metadata(hosts.worker1.cpu, a)") # POST # bad key request = {'operations': '[{ "metric": "hosts.worker1.cpu", "key": "a", "value": "b" }]'} response = self.client.post(url, request) self.assertEqual(response.status_code, 200) data = json.loads(response.content) self.assertEqual(data['hosts.worker1.cpu']['error'], "Unexpected error occurred in bulk CarbonLink.set_metadata(hosts.worker1.cpu)")
def create_temporary(request): state = str( json.dumps( json.loads( request.POST['state'] ) ) ) i = 0 while True: name = "temporary-%d" % i try: Dashboard.objects.get(name=name) except Dashboard.DoesNotExist: dashboard = Dashboard.objects.create(name=name, state=state) break else: i += 1 return json_response( dict(name=dashboard.name) )
def create_temporary(request): state = str(json.dumps(json.loads(request.POST['state']))) i = 0 while True: name = "temporary-%d" % i try: Dashboard.objects.get(name=name) except Dashboard.DoesNotExist: dashboard = Dashboard.objects.create(name=name, state=state) break else: i += 1 return json_response(dict(name=dashboard.name))
def save_template(request, name): # Deserialize and reserialize as a validation step state = str( json.dumps( json.loads( request.POST['state'] ) ) ) try: template = Template.objects.get(name=name) except Template.DoesNotExist: template = Template.objects.create(name=name) template.setState(state) template.save() else: template.setState(state) template.save(); return json_response( dict(success=True) )
def save(request, name): if 'change' not in getPermissions(request.user): return json_response( dict(error="Must be logged in with appropriate permissions to save") ) # Deserialize and reserialize as a validation step state = str( json.dumps( json.loads( request.POST['state'] ) ) ) try: dashboard = Dashboard.objects.get(name=name) except Dashboard.DoesNotExist: dashboard = Dashboard.objects.create(name=name, state=state) else: dashboard.state = state dashboard.save(); return json_response( dict(success=True) )
def template(request, name, val): template_conf_missing = False try: config.check() except OSError as e: if e.errno == errno.ENOENT: template_conf_missing = True else: raise initialError = None debug = request.GET.get('debug', False) theme = request.GET.get('theme', config.ui_config['theme']) css_file = finders.find('css/dashboard-%s.css' % theme) if css_file is None: initialError = "Invalid theme '%s'" % theme theme = config.ui_config['theme'] context = { 'schemes_json' : json.dumps(config.schemes), 'ui_config_json' : json.dumps(config.ui_config), 'jsdebug' : debug or settings.JAVASCRIPT_DEBUG, 'debug' : debug, 'theme' : theme, 'initialError' : initialError, 'querystring' : json.dumps( dict( request.GET.items() ) ), 'template_conf_missing' : template_conf_missing, 'userName': '', 'permissions': json.dumps(getPermissions(request.user)), 'permissionsUnauthenticated': json.dumps(getPermissions(None)) } user = request.user if user: context['userName'] = user.username try: template = Template.objects.get(name=name) except Template.DoesNotExist: context['initialError'] = "Template '%s' does not exist." % name else: state = json.loads(template.loadState(val)) state['name'] = '%s/%s' % (name, val) context['initialState'] = json.dumps(state) return render_to_response("dashboard.html", context)
def save_template(request, name, key): if 'change' not in getPermissions(request.user): return json_response( dict(error="Must be logged in with appropriate permissions to save the template") ) # Deserialize and reserialize as a validation step state = str( json.dumps( json.loads( request.POST['state'] ) ) ) try: template = Template.objects.get(name=name) except Template.DoesNotExist: template = Template.objects.create(name=name) template.setState(state) template.save() else: template.setState(state, key) template.save(); return json_response( dict(success=True) )
def setState(self, state): def replace_hostid(s): if isinstance(s, unicode): s = self.__class__.metric_path_re.sub('__HOST_ID__.', s) return s def update_graph(graph): graph_opts = graph[1] graph_opts['target'] = [replace_hostid(s) for s in graph_opts['target']] return [replace_hostid(graph[0]), graph_opts, replace_hostid(graph[2])] # Parse JSON here and replace first five elements of target with __HOST_ID__ parsed_state = json.loads(state) for i, graph in enumerate(parsed_state['graphs']): parsed_state['graphs'][i] = update_graph(graph) self.state = json.dumps(parsed_state)