def api_endpoints(): ret = { "ok": False, "msg": "", "data": [], } q = request.args.get("q") or "" raw_tag = request.args.get("tags") or "" tags = raw_tag and [x.strip() for x in raw_tag.split(",")] or [] limit = int(request.args.get("limit") or 100) if not q and not tags: ret["msg"] = "no query params given" return json.dumps(ret) endpoints = [] if tags and q: endpoint_ids = TagEndpoint.get_endpoint_ids(tags, limit=limit) or [] endpoints = Endpoint.search_in_ids(q.split(), endpoint_ids) elif tags: endpoint_ids = TagEndpoint.get_endpoint_ids(tags, limit=limit) or [] endpoints = Endpoint.gets(endpoint_ids) elif q == 'all': endpoints = Endpoint.search([], limit=limit) else: endpoints = Endpoint.search(q.split(), limit=limit) endpoints_str = [x.endpoint for x in endpoints] endpoints_str.sort() ret['data'] = endpoints_str ret['ok'] = True return json.dumps(ret)
def api_endpoints(): ret = { "ok": False, "msg": "", "data": [], } q = request.args.get("q") or "" raw_tag = request.args.get("tags") or "" tags = raw_tag and [x.strip() for x in raw_tag.split(",")] or [] limit = int(request.args.get("limit") or 100) if not q and not tags: ret["msg"] = "no query params given" return json.dumps(ret) endpoints = [] if tags and q: endpoint_ids = TagEndpoint.get_endpoint_ids(tags, limit=limit) or [] endpoints = Endpoint.search_in_ids(q.split(), endpoint_ids) elif tags: endpoint_ids = TagEndpoint.get_endpoint_ids(tags, limit=limit) or [] endpoints = Endpoint.gets(endpoint_ids) else: endpoints = Endpoint.search(q.split(), limit=limit) endpoints_str = [x.endpoint for x in endpoints] endpoints_str.sort() ret['data'] = endpoints_str ret['ok'] = True return json.dumps(ret)
def api_get_counters(): ret = { "ok": False, "msg": "", "data": [], } endpoints = request.form.get("endpoints") or "" endpoints = endpoints and json.loads(endpoints) q = request.form.get("q") or "" limit = int(request.form.get("limit") or 100) nethelp = request.form.get("nethelp") or "" if not (endpoints or q): ret['msg'] = "no endpoints or counter given" return json.dumps(ret) endpoint_objs = Endpoint.gets_by_endpoint(endpoints) endpoint_ids = [x.id for x in endpoint_objs] group_ids = [] if not endpoint_ids: group_objs = Group.gets_by_group(endpoints) group_ids = [x.id for x in group_objs] grouphost_objs = GroupHost.search(group_ids) host_ids = [x.hostId for x in grouphost_objs] host_objs = Host.search(host_ids) host_names = [x.name for x in host_objs] endpoint_objs = Endpoint.gets_by_endpoint(host_names) endpoint_ids = [x.id for x in endpoint_objs] if not endpoint_ids: ret['msg'] = "no endpoints in graph" return json.dumps(ret) qs = q.split() if nethelp == "true": ecs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids, limit=100) elif len(group_ids) > 0: limit = 5000 if len(qs) > 0: ecs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids, limit=limit) else: ecs = EndpointCounter.gets_by_endpoint_ids(endpoint_ids, limit=limit) if not ecs: ret["msg"] = "no counters in graph" return json.dumps(ret) counters_map = {} for x in ecs: counters_map[x.counter] = [x.counter, x.type_, x.step] sorted_counters = sorted(counters_map.keys()) sorted_values = [counters_map[x] for x in sorted_counters] ret['data'] = sorted_values ret['ok'] = True return json.dumps(ret)
def api_ztree(): ret = { "ok":False, "msg":"", "dara":[], } if request.method == "POST": pid = request.form.get("id") ztree = Ztree.get_name_by_id(pid) grp = list() for x in ztree: grp.append(x.name) filter = [(".").join(grp)] hosts = Endpoint.search(filter,limit=1000000) data = [] id = int(pid + "100000") for x in hosts: data.append({"id":str(id),"pid":pid,"name":x.endpoint,"isparent":"true"}) id += 1 return json.dumps(data) ztrees = Ztree.gets() ztrees_str = [] for x in ztrees: ztree_one = [x.id,x.pid,x.name,x.isparent] ztrees_str.append(ztree_one) ret['data'] = ztrees_str ret['ok'] = True return json.dumps(ret)
def generate_graph_urls(graph, start, end): print today_date_str(), "generate_graph_urls_start", graph.counters counters = graph.counters or [] if not counters: return [] endpoint_list = graph.hosts or [] if not endpoint_list: return [] endpoint_objs = Endpoint.gets_by_endpoint(endpoint_list) print today_date_str(), "generate_graph_urls_endpoint_objs" if not endpoint_objs: return [] endpoint_ids = [x.id for x in endpoint_objs] counters = [] print today_date_str(), len(graph.counters) for c in graph.counters: print today_date_str(), "c", c if c.find("metric=") == -1: counters.append(c) else: metric = "" tags = [] qs = [] c = c.strip() for q in c.split(): q = q.strip() if q.startswith("metric="): metric = q.replace("metric=", "^", 1) qs.append(metric) else: qs.append(q) tags.append(q) print "counter_objs_start" counter_objs = EndpointCounter.search_in_endpoint_ids( qs, endpoint_ids[:], limit=100) print "counter_objs", counter_objs if not counter_objs: continue for co in counter_objs: if not re.search('^%s(/|$)' % metric, co.counter): continue matched = True for tag in tags: if not re.search('(/|,)%s(,|$)' % tag, co.counter): matched = False break if not matched: continue counters.append(co.counter) if not counters: return [] counters = sorted(list(set(counters))) return _generate_graph_urls(graph, counters, endpoint_list, start, end)
def qryOptions(): options = {} options['hosts'] = Endpoint.search([], limit=limit) ids = [] for ep in options['hosts']: ids.append(ep.id) options['counters'] = EndpointCounter.gets_by_endpoint_ids(ids[0:1], limit=limit) return options
def api_get_counters(): ret = { "ok": False, "msg": "", "data": [], "tags":[], } endpoints = request.form.get("endpoints") or "" endpoints = endpoints and json.loads(endpoints) q = request.form.get("q") or "" limit = int(request.form.get("limit") or 100) if not (endpoints or q): ret['msg'] = "no endpoints or counter given" return json.dumps(ret) endpoint_objs = Endpoint.gets_by_endpoint(endpoints) endpoint_ids = [x.id for x in endpoint_objs] if not endpoint_ids: ret['msg'] = "no endpoints in graph" return json.dumps(ret) if q: qs = q.split() ecs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids, limit=limit) else: ecs = EndpointCounter.gets_by_endpoint_ids(endpoint_ids, limit=limit) if not ecs: ret["msg"] = "no counters in graph" return json.dumps(ret) counters_map = {} tags = ["notag"] for x in ecs: if x.counter.find("=") > 0: end = x.counter.rfind("/", 0,x.counter.find("=")) metric = x.counter[0:end] tag = x.counter[end+1:] tags.append(tag) else: tag = "" tags.append(tag) metric = x.counter if counters_map.has_key(metric): if tag not in counters_map[metric]: counters_map[metric] += [tag] else: counters_map[metric] = [metric,x.type_,x.step,tag] sorted_counters = sorted(counters_map.keys()) sorted_values = [counters_map[x] for x in sorted_counters] ret['data'] = sorted_values ret['ok'] = True ret['tags'] = filter(None,list(set(tags))) return json.dumps(ret)
def generate_graph_urls(graph, start, end): counters = graph.counters or [] if not counters: return [] endpoint_list = graph.hosts or [] if not endpoint_list: return [] endpoint_objs = Endpoint.gets_by_endpoint(endpoint_list) if not endpoint_objs: return [] endpoint_ids = [x.id for x in endpoint_objs] counters = [] for c in graph.counters: if c.find("metric=") == -1: counters.append(c) else: metric="" tags = [] qs = [] c = c.strip() for q in c.split(): q = q.strip() if q.startswith("metric="): metric = q.replace("metric=", "", 1) qs.append(metric) else: qs.append(q) tags.append(q) counter_objs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids[:], limit=100) if not counter_objs: continue for co in counter_objs: if not re.search('^%s(/|$)' %metric, co.counter): continue matched = True for tag in tags: if not re.search('(/|,)%s(,|$)' %tag, co.counter): matched = False break if not matched: continue counters.append(co.counter) if not counters: return [] counters = sorted(list(set(counters))) return _generate_graph_urls(graph, counters, endpoint_list, start, end)
def get_counterscore(): ret = { "ok": False, "msg": "", "data": [], } cluster = request.form.get("cluster") or "" counter = request.form.get("counter") or "" endpoints = Endpoint.search_endpoint_by_cluster(cluster) score = 100 for x in endpoints: method = "POST" handler = urllib2.HTTPHandler() opener = urllib2.build_opener(handler) url = config.QUERY_ADDR + "/graph/last" port = 0 p = [] if x.id!="None": port = x.id else: port = 3306 q = { "endpoint": x.endpoint, "counter": (counter + "%s")%port } p.append(q) request1 = urllib2.Request(url, data=json.dumps(p)) request1.add_header("Content-Type",'application/json') request1.get_method = lambda: method try: connection = opener.open(request1) except urllib2.HTTPError,e: connection = e # check. Substitute with appropriate HTTP code. if connection.code == 200: msg = connection.read() # print ("msg===>>>" + msg) jsonmsg = json.loads(msg) if jsonmsg[0]["value"]: if jsonmsg[0]["value"]["value"]: f = float(jsonmsg[0]["value"]["value"]) if f > 5: score = score -10 ret['ok'] = True else: print '{"err":1,"msg":"%s"}' % connection ret['ok'] = False
def api_endpoints_type(type): ret = { "ok": False, "msg": "", "data": [], } endpoints = Endpoint.search_type(type) endpoints_str = [x.endpoint for x in endpoints] endpoints_str.sort() ret['data'] = endpoints_str ret['ok'] = True return json.dumps(ret)
def get_endpoints_by_cluster(cluster): ret = { "ok": False, "msg": "", "data": [], } endpoints = Endpoint.search_endpoint_by_cluster(cluster) endpoints_str = [x.endpoint for x in endpoints] endpoints_str.sort() ret['data'] = endpoints_str ret['ok'] = True return json.dumps(ret)
def api_endpoints_cluster(type): ret = { "ok": False, "msg": "", "data": [], } clusters = Endpoint.search_cluster(type) clusters_str = [x.endpoint for x in clusters] clusters_str.sort() ret['data'] = clusters_str ret['ok'] = True return json.dumps(ret)
def dash_graph_add(sid): all_screens = DashboardScreen.gets_all() top_screens = [x for x in all_screens if x.pid == '0'] children = [] for t in top_screens: children.append([x for x in all_screens if x.pid == t.id]) screen = DashboardScreen.get(sid) if not screen: abort(404, "no screen") pscreen = DashboardScreen.get(screen.pid) if request.method == "POST": title = request.form.get("title") hosts = request.form.get("hosts", "").strip() hosts = hosts and hosts.split("\n") or [] hosts = [x.strip() for x in hosts] counters = request.form.get("counters", "").strip() counters = counters and counters.split("\n") or [] counters = [x.strip() for x in counters] timespan = int(request.form.get("timespan", 3600)) graph_type = request.form.get("graph_type", 'h') method = request.form.get("method", '').upper() position = int(request.form.get("position", 0)) graph = DashboardGraph.add(title, hosts, counters, sid, timespan, graph_type, method, position) return redirect("/screen/%s" % sid) else: limit = 10000 gid = request.args.get("gid") graph = gid and DashboardGraph.get(gid) options = {} options['hosts'] = Endpoint.search(''.split(), limit=limit) ids = [] for ep in options['hosts']: ids.append(ep.id) options['counters'] = EndpointCounter.gets_by_endpoint_ids(ids[0:1], limit=limit) return render_template("screen/graph_add.html", config=config, **locals())
def get_groupscore_by_cluster(cluster): ret = { "ok": False, "msg": "", "data": [], } endpoints = Endpoint.search_agent_endpoint_by_cluster(cluster) score = 0 for endpoint in endpoints: p = [] q = { "endpoint": endpoint.endpoint, "counter": "net.port.listen/port=%s"%endpoint.id } p.append(q) method = "POST" handler = urllib2.HTTPHandler() opener = urllib2.build_opener(handler) url = config.QUERY_ADDR + "/graph/last" # print ("post data==>>>%s"%json.dumps(p)) request = urllib2.Request(url, data=json.dumps(p)) request.add_header("Content-Type", "application/json") request.get_method = lambda: method try: connection = opener.open(request) except urllib2.HTTPError,e: connection = e # check. Substitute with appropriate HTTP code. if connection.code == 200: msg = connection.read() jsonmsg = json.loads(msg) value = jsonmsg[0]["value"]["value"] score = score + int(value) #j = { # "endpoint": endpoint.endpoint, # "value": score #} ret['ok'] = True else: print '{"err":1,"msg":"%s"}' % connection ret['ok'] = False
def api_get_counters(): RefreshCounterDesp() ret = { "ok": False, "msg": "", "data": [], } endpoints = request.form.get("endpoints") or "" endpoints = endpoints and json.loads(endpoints) q = request.form.get("q") or "" limit = int(request.form.get("limit") or 100) if not (endpoints or q): ret['msg'] = "no endpoints or counter given" return json.dumps(ret) endpoint_objs = Endpoint.gets_by_endpoint(endpoints) endpoint_ids = [x.id for x in endpoint_objs] if not endpoint_ids: ret['msg'] = "no endpoints in graph" return json.dumps(ret) if q: qs = q.split() ecs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids, limit=limit) else: ecs = EndpointCounter.gets_by_endpoint_ids(endpoint_ids, limit=limit) if not ecs: ret["msg"] = "no counters in graph" return json.dumps(ret) counters_map = {} for x in ecs: counters_map[x.counter] = [x.counter, x.type_, x.step, x.desp] sorted_counters = sorted(counters_map.keys()) sorted_values = [counters_map[x] for x in sorted_counters] ret['data'] = sorted_values ret['ok'] = True return json.dumps(ret)
def get_memscore_by_cluster(cluster): ret = { "ok": False, "msg": "", "data": [], } endpoints = Endpoint.search_httpapi_by_cluster(cluster) score = 100 for x in endpoints: url = x.endpoint method = "GET" handler = urllib2.HTTPHandler() opener = urllib2.build_opener(handler) url = url + "/page/memory" request = urllib2.Request(url) request.add_header("Content-Type",'application/json') request.get_method = lambda: method try: connection = opener.open(request) except urllib2.HTTPError,e: connection = e # check. Substitute with appropriate HTTP code. if connection.code == 200: msg = connection.read() # print ("msg===>>>" + msg) jsonmsg = json.loads(msg) if jsonmsg["msg"]=="success": if jsonmsg["data"]: x = jsonmsg["data"][1] y = jsonmsg["data"][0] a = x/float(y) if a > 0.2: score = score - 10 else: score = score - 20 ret['ok'] = True else: print '{"err":1,"msg":"%s"}' % connection ret['ok'] = False
def api_get_counters(): ret = { "ok": False, "msg": "", "data": [], } endpoints = request.form.get("endpoints") or "" endpoints = endpoints and json.loads(endpoints) q = request.form.get("q") or "" limit = int(request.form.get("limit") or 100) if not (endpoints or q): ret['msg'] = "no endpoints or counter given" return json.dumps(ret) endpoint_objs = Endpoint.gets_by_endpoint(endpoints) endpoint_ids = [x.id for x in endpoint_objs] if not endpoint_ids: ret['msg'] = "no endpoints in graph" return json.dumps(ret) if q: qs = q.split() ecs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids, limit=limit) else: ecs = EndpointCounter.gets_by_endpoint_ids(endpoint_ids, limit=limit) if not ecs: ret["msg"] = "no counters in graph" return json.dumps(ret) counters_map = {} for x in ecs: counters_map[x.counter] = [x.counter, x.type_, x.step] sorted_counters = sorted(counters_map.keys()) sorted_values = [counters_map[x] for x in sorted_counters] ret['data'] = sorted_values ret['ok'] = True return json.dumps(ret)
def get_endpoint_detail_charts(): counters = [] endpoints = [] counters0 = request.form.getlist("counters[]") or [] graph_type = request.form.get("graph_type") or GRAPH_TYPE_HOST endpoint0 = request.form.get("endpoint") or "" qtype = request.form.get("type") or "" endpointlist = Endpoint.search_agent_and_httpapi_by_endpoint(endpoint0) for endpoint in endpointlist: endpoints.append(endpoint.endpoint) for counter in counters0: if counter == "port": counters.append("net.port.listen/port=" + endpoint.id) elif counter == "memory": counters.append("mem.memfree.percent") elif counter == "df": counters.append("df.statistics.used.percent") else: if (qtype == "mysql" or qtype == "redis"): counters.append(counter + endpoint.id) else: counters.append(counter) endpoints.append(endpoint0) print endpoints print counters id_ = TmpGraph.add(endpoints, counters) ret = { "ok": False, "id": id_, "params": { "graph_type": graph_type, }, } if id_: ret['ok'] = True return json.dumps(ret)
def api_get_counters(): ret = { "ok": False, "msg": "", "data": [], } data = request.json or "" type = data['type'] endpoints = [] filter = "" containers = [] if not type: ret['msg'] = "no type given" return json.dumps(ret) if type == "node": endpoints = data['data'] filter = data['filter'] elif type == "pod": filter = data['filter'] for pod in data['data']: for container in pod['containers']: node = container['hostname'] containers.append(container['containerId']) if len(endpoints) == 0: endpoints.append(node) else: p = 0 for endpoint in endpoints: if endpoint == node: break else: p = p + 1 if p != len(endpoints): endpoints.append(node) elif type == "container": filter = data['filter'] for container in data['data']: node = container['hostname'] containers.append(container['containerId']) if len(endpoints) == 0: endpoints.append(node) else: p = 0 for endpoint in endpoints: if endpoint == node: break else: p = p + 1 if p != len(endpoints): endpoints.append(node) endpoint_objs = Endpoint.gets_by_endpoint(endpoints) endpoint_ids = [x.id for x in endpoint_objs] if not endpoint_ids: ret['msg'] = "no endpoints in graph" return json.dumps(ret) if filter: filters = filter.split() ecs = EndpointCounter.search_in_endpoint_ids(filters, endpoint_ids) else: ecs = EndpointCounter.gets_by_endpoint_ids(endpoint_ids) if not ecs: ret["msg"] = "no counters in graph" return json.dumps(ret) counters_map = {} for x in ecs: if type == 'node': if 'id=' not in x.counter: counters_map[x.counter] = [x.counter, x.type_, x.step] elif type == 'pod' or type == 'container': if 'id=' in x.counter: if x.counter.split('/')[1].split('=')[1] in containers: counters_map[x.counter] = [x.counter, x.type_, x.step] sorted_counters = sorted(counters_map.keys()) sorted_values = [counters_map[x] for x in sorted_counters] ret['data'] = sorted_values ret['ok'] = True return json.dumps(ret)
def get_endpoint_detail(): ret = { "ok": False, "msg": "", "data": [], } endpoint0 = request.form.get("endpoint") or "" qparam = request.form.getlist("q[]") or [] print qparam qtype = request.form.get("type") or "" endpoints = Endpoint.search_agent_and_httpapi_by_endpoint(endpoint0) for endpoint in endpoints: if 'port' in qparam: p = [] q = { "endpoint": endpoint.endpoint, "counter": "net.port.listen/port=%s"%endpoint.id } p.append(q) method = "POST" handler = urllib2.HTTPHandler() opener = urllib2.build_opener(handler) url = config.QUERY_ADDR + "/graph/last" request1 = urllib2.Request(url, data=json.dumps(p)) request1.add_header("Content-Type", "application/json") request1.get_method = lambda: method try: connection = opener.open(request1) except urllib2.HTTPError,e: connection = e # check. Substitute with appropriate HTTP code. if connection.code == 200: msg = connection.read() jsonmsg = json.loads(msg) value = jsonmsg[0]["value"]["value"] score = int(value) j = [] j.append('port') j.append(score) ret['data'].append(j) ret['ok'] = True qparam.remove('port') else: print '{"err":1,"msg":"%s"}' % connection ret['ok'] = False # memory if (ret['ok'] and 'memory' in qparam): http_api = endpoint.ts method = "GET" handler = urllib2.HTTPHandler() opener = urllib2.build_opener(handler) request2 = urllib2.Request(http_api + "/page/memory") request2.add_header("Content-Type",'application/json') request2.get_method = lambda: method try: connection = opener.open(request2) except urllib2.HTTPError,e: connection = e # check. Substitute with appropriate HTTP code. if connection.code == 200: msg = connection.read() jsonmsg = json.loads(msg) if jsonmsg["msg"]=="success": if jsonmsg["data"]: x = jsonmsg["data"][1] y = jsonmsg["data"][0] a = "%.2f"%(x*100/float(y)) j = [] j.append('memory') j.append(a + '%') ret['data'].append(j) qparam.remove('memory') else: ret['ok'] = False break else: print '{"err":1,"msg":"%s"}' % connection ret['ok'] = False
def index(): raw_endpoints = Endpoint.search(""); endpoints = [e.endpoint for e in raw_endpoints] endpoints.sort() return render_template("index.html", **locals())