def api_endpoints():
ret = {
"ok": False,
"msg": "",
"data": [],
}
q = request.args.get("q") or ""
raw_tag = request.args.get("tags") or ""
tags = raw_tag and [x.strip() for x in raw_tag.split(",")] or []
limit = int(request.args.get("limit") or 100)
if not q and not tags:
ret["msg"] = "no query params given"
return json.dumps(ret)
endpoints = []
if tags and q:
endpoint_ids = TagEndpoint.get_endpoint_ids(tags, limit=limit) or []
endpoints = Endpoint.search_in_ids(q.split(), endpoint_ids)
elif tags:
endpoint_ids = TagEndpoint.get_endpoint_ids(tags, limit=limit) or []
endpoints = Endpoint.gets(endpoint_ids)
elif q == 'all':
endpoints = Endpoint.search([], limit=limit)
else:
endpoints = Endpoint.search(q.split(), limit=limit)
endpoints_str = [x.endpoint for x in endpoints]
endpoints_str.sort()
ret['data'] = endpoints_str
ret['ok'] = True
return json.dumps(ret)
def api_endpoints():
ret = {
"ok": False,
"msg": "",
"data": [],
}
q = request.args.get("q") or ""
raw_tag = request.args.get("tags") or ""
tags = raw_tag and [x.strip() for x in raw_tag.split(",")] or []
limit = int(request.args.get("limit") or 100)
if not q and not tags:
ret["msg"] = "no query params given"
return json.dumps(ret)
endpoints = []
if tags and q:
endpoint_ids = TagEndpoint.get_endpoint_ids(tags, limit=limit) or []
endpoints = Endpoint.search_in_ids(q.split(), endpoint_ids)
elif tags:
endpoint_ids = TagEndpoint.get_endpoint_ids(tags, limit=limit) or []
endpoints = Endpoint.gets(endpoint_ids)
else:
endpoints = Endpoint.search(q.split(), limit=limit)
endpoints_str = [x.endpoint for x in endpoints]
endpoints_str.sort()
ret['data'] = endpoints_str
ret['ok'] = True
return json.dumps(ret)
def api_get_counters():
ret = {
"ok": False,
"msg": "",
"data": [],
}
endpoints = request.form.get("endpoints") or ""
endpoints = endpoints and json.loads(endpoints)
q = request.form.get("q") or ""
limit = int(request.form.get("limit") or 100)
nethelp = request.form.get("nethelp") or ""
if not (endpoints or q):
ret['msg'] = "no endpoints or counter given"
return json.dumps(ret)
endpoint_objs = Endpoint.gets_by_endpoint(endpoints)
endpoint_ids = [x.id for x in endpoint_objs]
group_ids = []
if not endpoint_ids:
group_objs = Group.gets_by_group(endpoints)
group_ids = [x.id for x in group_objs]
grouphost_objs = GroupHost.search(group_ids)
host_ids = [x.hostId for x in grouphost_objs]
host_objs = Host.search(host_ids)
host_names = [x.name for x in host_objs]
endpoint_objs = Endpoint.gets_by_endpoint(host_names)
endpoint_ids = [x.id for x in endpoint_objs]
if not endpoint_ids:
ret['msg'] = "no endpoints in graph"
return json.dumps(ret)
qs = q.split()
if nethelp == "true":
ecs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids, limit=100)
elif len(group_ids) > 0:
limit = 5000
if len(qs) > 0:
ecs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids, limit=limit)
else:
ecs = EndpointCounter.gets_by_endpoint_ids(endpoint_ids, limit=limit)
if not ecs:
ret["msg"] = "no counters in graph"
return json.dumps(ret)
counters_map = {}
for x in ecs:
counters_map[x.counter] = [x.counter, x.type_, x.step]
sorted_counters = sorted(counters_map.keys())
sorted_values = [counters_map[x] for x in sorted_counters]
ret['data'] = sorted_values
ret['ok'] = True
return json.dumps(ret)
def api_ztree():
ret = {
"ok":False,
"msg":"",
"dara":[],
}
if request.method == "POST":
pid = request.form.get("id")
ztree = Ztree.get_name_by_id(pid)
grp = list()
for x in ztree:
grp.append(x.name)
filter = [(".").join(grp)]
hosts = Endpoint.search(filter,limit=1000000)
data = []
id = int(pid + "100000")
for x in hosts:
data.append({"id":str(id),"pid":pid,"name":x.endpoint,"isparent":"true"})
id += 1
return json.dumps(data)
ztrees = Ztree.gets()
ztrees_str = []
for x in ztrees:
ztree_one = [x.id,x.pid,x.name,x.isparent]
ztrees_str.append(ztree_one)
ret['data'] = ztrees_str
ret['ok'] = True
return json.dumps(ret)
def generate_graph_urls(graph, start, end):
print today_date_str(), "generate_graph_urls_start", graph.counters
counters = graph.counters or []
if not counters:
return []
endpoint_list = graph.hosts or []
if not endpoint_list:
return []
endpoint_objs = Endpoint.gets_by_endpoint(endpoint_list)
print today_date_str(), "generate_graph_urls_endpoint_objs"
if not endpoint_objs:
return []
endpoint_ids = [x.id for x in endpoint_objs]
counters = []
print today_date_str(), len(graph.counters)
for c in graph.counters:
print today_date_str(), "c", c
if c.find("metric=") == -1:
counters.append(c)
else:
metric = ""
tags = []
qs = []
c = c.strip()
for q in c.split():
q = q.strip()
if q.startswith("metric="):
metric = q.replace("metric=", "^", 1)
qs.append(metric)
else:
qs.append(q)
tags.append(q)
print "counter_objs_start"
counter_objs = EndpointCounter.search_in_endpoint_ids(
qs, endpoint_ids[:], limit=100)
print "counter_objs", counter_objs
if not counter_objs:
continue
for co in counter_objs:
if not re.search('^%s(/|$)' % metric, co.counter):
continue
matched = True
for tag in tags:
if not re.search('(/|,)%s(,|$)' % tag, co.counter):
matched = False
break
if not matched:
continue
counters.append(co.counter)
if not counters:
return []
counters = sorted(list(set(counters)))
return _generate_graph_urls(graph, counters, endpoint_list, start, end)
def qryOptions():
options = {}
options['hosts'] = Endpoint.search([], limit=limit)
ids = []
for ep in options['hosts']:
ids.append(ep.id)
options['counters'] = EndpointCounter.gets_by_endpoint_ids(ids[0:1], limit=limit)
return options
def api_get_counters():
ret = {
"ok": False,
"msg": "",
"data": [],
"tags":[],
}
endpoints = request.form.get("endpoints") or ""
endpoints = endpoints and json.loads(endpoints)
q = request.form.get("q") or ""
limit = int(request.form.get("limit") or 100)
if not (endpoints or q):
ret['msg'] = "no endpoints or counter given"
return json.dumps(ret)
endpoint_objs = Endpoint.gets_by_endpoint(endpoints)
endpoint_ids = [x.id for x in endpoint_objs]
if not endpoint_ids:
ret['msg'] = "no endpoints in graph"
return json.dumps(ret)
if q:
qs = q.split()
ecs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids, limit=limit)
else:
ecs = EndpointCounter.gets_by_endpoint_ids(endpoint_ids, limit=limit)
if not ecs:
ret["msg"] = "no counters in graph"
return json.dumps(ret)
counters_map = {}
tags = ["notag"]
for x in ecs:
if x.counter.find("=") > 0:
end = x.counter.rfind("/", 0,x.counter.find("="))
metric = x.counter[0:end]
tag = x.counter[end+1:]
tags.append(tag)
else:
tag = ""
tags.append(tag)
metric = x.counter
if counters_map.has_key(metric):
if tag not in counters_map[metric]:
counters_map[metric] += [tag]
else:
counters_map[metric] = [metric,x.type_,x.step,tag]
sorted_counters = sorted(counters_map.keys())
sorted_values = [counters_map[x] for x in sorted_counters]
ret['data'] = sorted_values
ret['ok'] = True
ret['tags'] = filter(None,list(set(tags)))
return json.dumps(ret)
def generate_graph_urls(graph, start, end):
counters = graph.counters or []
if not counters:
return []
endpoint_list = graph.hosts or []
if not endpoint_list:
return []
endpoint_objs = Endpoint.gets_by_endpoint(endpoint_list)
if not endpoint_objs:
return []
endpoint_ids = [x.id for x in endpoint_objs]
counters = []
for c in graph.counters:
if c.find("metric=") == -1:
counters.append(c)
else:
metric=""
tags = []
qs = []
c = c.strip()
for q in c.split():
q = q.strip()
if q.startswith("metric="):
metric = q.replace("metric=", "", 1)
qs.append(metric)
else:
qs.append(q)
tags.append(q)
counter_objs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids[:], limit=100)
if not counter_objs:
continue
for co in counter_objs:
if not re.search('^%s(/|$)' %metric, co.counter):
continue
matched = True
for tag in tags:
if not re.search('(/|,)%s(,|$)' %tag, co.counter):
matched = False
break
if not matched:
continue
counters.append(co.counter)
if not counters:
return []
counters = sorted(list(set(counters)))
return _generate_graph_urls(graph, counters, endpoint_list, start, end)
def get_counterscore():
ret = {
"ok": False,
"msg": "",
"data": [],
}
cluster = request.form.get("cluster") or ""
counter = request.form.get("counter") or ""
endpoints = Endpoint.search_endpoint_by_cluster(cluster)
score = 100
for x in endpoints:
method = "POST"
handler = urllib2.HTTPHandler()
opener = urllib2.build_opener(handler)
url = config.QUERY_ADDR + "/graph/last"
port = 0
p = []
if x.id!="None":
port = x.id
else:
port = 3306
q = {
"endpoint": x.endpoint,
"counter": (counter + "%s")%port
}
p.append(q)
request1 = urllib2.Request(url, data=json.dumps(p))
request1.add_header("Content-Type",'application/json')
request1.get_method = lambda: method
try:
connection = opener.open(request1)
except urllib2.HTTPError,e:
connection = e
# check. Substitute with appropriate HTTP code.
if connection.code == 200:
msg = connection.read()
# print ("msg===>>>" + msg)
jsonmsg = json.loads(msg)
if jsonmsg[0]["value"]:
if jsonmsg[0]["value"]["value"]:
f = float(jsonmsg[0]["value"]["value"])
if f > 5:
score = score -10
ret['ok'] = True
else:
print '{"err":1,"msg":"%s"}' % connection
ret['ok'] = False
def api_endpoints_type(type):
ret = {
"ok": False,
"msg": "",
"data": [],
}
endpoints = Endpoint.search_type(type)
endpoints_str = [x.endpoint for x in endpoints]
endpoints_str.sort()
ret['data'] = endpoints_str
ret['ok'] = True
return json.dumps(ret)
def get_endpoints_by_cluster(cluster):
ret = {
"ok": False,
"msg": "",
"data": [],
}
endpoints = Endpoint.search_endpoint_by_cluster(cluster)
endpoints_str = [x.endpoint for x in endpoints]
endpoints_str.sort()
ret['data'] = endpoints_str
ret['ok'] = True
return json.dumps(ret)
def api_endpoints_cluster(type):
ret = {
"ok": False,
"msg": "",
"data": [],
}
clusters = Endpoint.search_cluster(type)
clusters_str = [x.endpoint for x in clusters]
clusters_str.sort()
ret['data'] = clusters_str
ret['ok'] = True
return json.dumps(ret)
def dash_graph_add(sid):
all_screens = DashboardScreen.gets_all()
top_screens = [x for x in all_screens if x.pid == '0']
children = []
for t in top_screens:
children.append([x for x in all_screens if x.pid == t.id])
screen = DashboardScreen.get(sid)
if not screen:
abort(404, "no screen")
pscreen = DashboardScreen.get(screen.pid)
if request.method == "POST":
title = request.form.get("title")
hosts = request.form.get("hosts", "").strip()
hosts = hosts and hosts.split("\n") or []
hosts = [x.strip() for x in hosts]
counters = request.form.get("counters", "").strip()
counters = counters and counters.split("\n") or []
counters = [x.strip() for x in counters]
timespan = int(request.form.get("timespan", 3600))
graph_type = request.form.get("graph_type", 'h')
method = request.form.get("method", '').upper()
position = int(request.form.get("position", 0))
graph = DashboardGraph.add(title, hosts, counters, sid, timespan,
graph_type, method, position)
return redirect("/screen/%s" % sid)
else:
limit = 10000
gid = request.args.get("gid")
graph = gid and DashboardGraph.get(gid)
options = {}
options['hosts'] = Endpoint.search(''.split(), limit=limit)
ids = []
for ep in options['hosts']:
ids.append(ep.id)
options['counters'] = EndpointCounter.gets_by_endpoint_ids(ids[0:1],
limit=limit)
return render_template("screen/graph_add.html",
config=config,
**locals())
def get_groupscore_by_cluster(cluster):
ret = {
"ok": False,
"msg": "",
"data": [],
}
endpoints = Endpoint.search_agent_endpoint_by_cluster(cluster)
score = 0
for endpoint in endpoints:
p = []
q = {
"endpoint": endpoint.endpoint,
"counter": "net.port.listen/port=%s"%endpoint.id
}
p.append(q)
method = "POST"
handler = urllib2.HTTPHandler()
opener = urllib2.build_opener(handler)
url = config.QUERY_ADDR + "/graph/last"
# print ("post data==>>>%s"%json.dumps(p))
request = urllib2.Request(url, data=json.dumps(p))
request.add_header("Content-Type", "application/json")
request.get_method = lambda: method
try:
connection = opener.open(request)
except urllib2.HTTPError,e:
connection = e
# check. Substitute with appropriate HTTP code.
if connection.code == 200:
msg = connection.read()
jsonmsg = json.loads(msg)
value = jsonmsg[0]["value"]["value"]
score = score + int(value)
#j = {
# "endpoint": endpoint.endpoint,
# "value": score
#}
ret['ok'] = True
else:
print '{"err":1,"msg":"%s"}' % connection
ret['ok'] = False
def api_get_counters():
RefreshCounterDesp()
ret = {
"ok": False,
"msg": "",
"data": [],
}
endpoints = request.form.get("endpoints") or ""
endpoints = endpoints and json.loads(endpoints)
q = request.form.get("q") or ""
limit = int(request.form.get("limit") or 100)
if not (endpoints or q):
ret['msg'] = "no endpoints or counter given"
return json.dumps(ret)
endpoint_objs = Endpoint.gets_by_endpoint(endpoints)
endpoint_ids = [x.id for x in endpoint_objs]
if not endpoint_ids:
ret['msg'] = "no endpoints in graph"
return json.dumps(ret)
if q:
qs = q.split()
ecs = EndpointCounter.search_in_endpoint_ids(qs, endpoint_ids, limit=limit)
else:
ecs = EndpointCounter.gets_by_endpoint_ids(endpoint_ids, limit=limit)
if not ecs:
ret["msg"] = "no counters in graph"
return json.dumps(ret)
counters_map = {}
for x in ecs:
counters_map[x.counter] = [x.counter, x.type_, x.step, x.desp]
sorted_counters = sorted(counters_map.keys())
sorted_values = [counters_map[x] for x in sorted_counters]
ret['data'] = sorted_values
ret['ok'] = True
return json.dumps(ret)
def get_memscore_by_cluster(cluster):
ret = {
"ok": False,
"msg": "",
"data": [],
}
endpoints = Endpoint.search_httpapi_by_cluster(cluster)
score = 100
for x in endpoints:
url = x.endpoint
method = "GET"
handler = urllib2.HTTPHandler()
opener = urllib2.build_opener(handler)
url = url + "/page/memory"
request = urllib2.Request(url)
request.add_header("Content-Type",'application/json')
request.get_method = lambda: method
try:
connection = opener.open(request)
except urllib2.HTTPError,e:
connection = e
# check. Substitute with appropriate HTTP code.
if connection.code == 200:
msg = connection.read()
# print ("msg===>>>" + msg)
jsonmsg = json.loads(msg)
if jsonmsg["msg"]=="success":
if jsonmsg["data"]:
x = jsonmsg["data"][1]
y = jsonmsg["data"][0]
a = x/float(y)
if a > 0.2:
score = score - 10
else:
score = score - 20
ret['ok'] = True
else:
print '{"err":1,"msg":"%s"}' % connection
ret['ok'] = False
def api_get_counters():
ret = {
"ok": False,
"msg": "",
"data": [],
}
endpoints = request.form.get("endpoints") or ""
endpoints = endpoints and json.loads(endpoints)
q = request.form.get("q") or ""
limit = int(request.form.get("limit") or 100)
if not (endpoints or q):
ret['msg'] = "no endpoints or counter given"
return json.dumps(ret)
endpoint_objs = Endpoint.gets_by_endpoint(endpoints)
endpoint_ids = [x.id for x in endpoint_objs]
if not endpoint_ids:
ret['msg'] = "no endpoints in graph"
return json.dumps(ret)
if q:
qs = q.split()
ecs = EndpointCounter.search_in_endpoint_ids(qs,
endpoint_ids,
limit=limit)
else:
ecs = EndpointCounter.gets_by_endpoint_ids(endpoint_ids, limit=limit)
if not ecs:
ret["msg"] = "no counters in graph"
return json.dumps(ret)
counters_map = {}
for x in ecs:
counters_map[x.counter] = [x.counter, x.type_, x.step]
sorted_counters = sorted(counters_map.keys())
sorted_values = [counters_map[x] for x in sorted_counters]
ret['data'] = sorted_values
ret['ok'] = True
return json.dumps(ret)
def get_endpoint_detail_charts():
counters = []
endpoints = []
counters0 = request.form.getlist("counters[]") or []
graph_type = request.form.get("graph_type") or GRAPH_TYPE_HOST
endpoint0 = request.form.get("endpoint") or ""
qtype = request.form.get("type") or ""
endpointlist = Endpoint.search_agent_and_httpapi_by_endpoint(endpoint0)
for endpoint in endpointlist:
endpoints.append(endpoint.endpoint)
for counter in counters0:
if counter == "port":
counters.append("net.port.listen/port=" + endpoint.id)
elif counter == "memory":
counters.append("mem.memfree.percent")
elif counter == "df":
counters.append("df.statistics.used.percent")
else:
if (qtype == "mysql" or qtype == "redis"):
counters.append(counter + endpoint.id)
else:
counters.append(counter)
endpoints.append(endpoint0)
print endpoints
print counters
id_ = TmpGraph.add(endpoints, counters)
ret = {
"ok": False,
"id": id_,
"params": {
"graph_type": graph_type,
},
}
if id_:
ret['ok'] = True
return json.dumps(ret)
def api_get_counters():
ret = {
"ok": False,
"msg": "",
"data": [],
}
data = request.json or ""
type = data['type']
endpoints = []
filter = ""
containers = []
if not type:
ret['msg'] = "no type given"
return json.dumps(ret)
if type == "node":
endpoints = data['data']
filter = data['filter']
elif type == "pod":
filter = data['filter']
for pod in data['data']:
for container in pod['containers']:
node = container['hostname']
containers.append(container['containerId'])
if len(endpoints) == 0:
endpoints.append(node)
else:
p = 0
for endpoint in endpoints:
if endpoint == node:
break
else:
p = p + 1
if p != len(endpoints):
endpoints.append(node)
elif type == "container":
filter = data['filter']
for container in data['data']:
node = container['hostname']
containers.append(container['containerId'])
if len(endpoints) == 0:
endpoints.append(node)
else:
p = 0
for endpoint in endpoints:
if endpoint == node:
break
else:
p = p + 1
if p != len(endpoints):
endpoints.append(node)
endpoint_objs = Endpoint.gets_by_endpoint(endpoints)
endpoint_ids = [x.id for x in endpoint_objs]
if not endpoint_ids:
ret['msg'] = "no endpoints in graph"
return json.dumps(ret)
if filter:
filters = filter.split()
ecs = EndpointCounter.search_in_endpoint_ids(filters, endpoint_ids)
else:
ecs = EndpointCounter.gets_by_endpoint_ids(endpoint_ids)
if not ecs:
ret["msg"] = "no counters in graph"
return json.dumps(ret)
counters_map = {}
for x in ecs:
if type == 'node':
if 'id=' not in x.counter:
counters_map[x.counter] = [x.counter, x.type_, x.step]
elif type == 'pod' or type == 'container':
if 'id=' in x.counter:
if x.counter.split('/')[1].split('=')[1] in containers:
counters_map[x.counter] = [x.counter, x.type_, x.step]
sorted_counters = sorted(counters_map.keys())
sorted_values = [counters_map[x] for x in sorted_counters]
ret['data'] = sorted_values
ret['ok'] = True
return json.dumps(ret)
def get_endpoint_detail():
ret = {
"ok": False,
"msg": "",
"data": [],
}
endpoint0 = request.form.get("endpoint") or ""
qparam = request.form.getlist("q[]") or []
print qparam
qtype = request.form.get("type") or ""
endpoints = Endpoint.search_agent_and_httpapi_by_endpoint(endpoint0)
for endpoint in endpoints:
if 'port' in qparam:
p = []
q = {
"endpoint": endpoint.endpoint,
"counter": "net.port.listen/port=%s"%endpoint.id
}
p.append(q)
method = "POST"
handler = urllib2.HTTPHandler()
opener = urllib2.build_opener(handler)
url = config.QUERY_ADDR + "/graph/last"
request1 = urllib2.Request(url, data=json.dumps(p))
request1.add_header("Content-Type", "application/json")
request1.get_method = lambda: method
try:
connection = opener.open(request1)
except urllib2.HTTPError,e:
connection = e
# check. Substitute with appropriate HTTP code.
if connection.code == 200:
msg = connection.read()
jsonmsg = json.loads(msg)
value = jsonmsg[0]["value"]["value"]
score = int(value)
j = []
j.append('port')
j.append(score)
ret['data'].append(j)
ret['ok'] = True
qparam.remove('port')
else:
print '{"err":1,"msg":"%s"}' % connection
ret['ok'] = False
# memory
if (ret['ok'] and 'memory' in qparam):
http_api = endpoint.ts
method = "GET"
handler = urllib2.HTTPHandler()
opener = urllib2.build_opener(handler)
request2 = urllib2.Request(http_api + "/page/memory")
request2.add_header("Content-Type",'application/json')
request2.get_method = lambda: method
try:
connection = opener.open(request2)
except urllib2.HTTPError,e:
connection = e
# check. Substitute with appropriate HTTP code.
if connection.code == 200:
msg = connection.read()
jsonmsg = json.loads(msg)
if jsonmsg["msg"]=="success":
if jsonmsg["data"]:
x = jsonmsg["data"][1]
y = jsonmsg["data"][0]
a = "%.2f"%(x*100/float(y))
j = []
j.append('memory')
j.append(a + '%')
ret['data'].append(j)
qparam.remove('memory')
else:
ret['ok'] = False
break
else:
print '{"err":1,"msg":"%s"}' % connection
ret['ok'] = False
def index():
raw_endpoints = Endpoint.search("");
endpoints = [e.endpoint for e in raw_endpoints]
endpoints.sort()
return render_template("index.html", **locals())