def getTables(self, db):
""" 返回指定数据库的所有表 """
with SQLite3(db) as client:
tables = yield client.read(
"SELECT name FROM sqlite_master WHERE type='table'")
raise Return([t[0] for t in tables if 'sqlite' not in t[0]])
def getTables(self, db, cluster):
""" 返回指定数据库的所有表 """
aliveTopics = instance.kafkaConfig[cluster]["topic"]
with SQLite3(db) as client:
tables = yield client.read("SELECT name FROM sqlite_master WHERE type='table'")
raise Return([t[0] for t in tables if t[0] in aliveTopics])
def flowCounter(self, db, table, partitions, category, group=None):
""" 计算实息流速 """
if not group:
# 计算cluster消息流速
sql = """
SELECT {0}
FROM `{1}`
WHERE category = '{2}'
AND group_name = (
SELECT group_name
FROM `{1}`
ORDER BY id DESC
LIMIT 1
) ORDER BY id DESC
LIMIT 2
"""
else:
# 计算topic/group消息流速
sql = """
SELECT {}
FROM `{}`
WHERE category = '{}'
AND group_name = '%s'
ORDER BY id DESC
LIMIT 2
""" % group
try:
with SQLite3(db) as client:
data = yield client.read(
sql.format(
",".join(["p{0}".format(p) for p in partitions]),
table,
category
)
)
flow = sum(map(lambda x:x[0]-x[1], zip(*data)))
except Exception:
raise Return(0)
raise Return(flow)
def getHighcharts(self, cluster, start, end):
""" 返回Hightchats数据 """
column = selectColumn(end - start)
db = os.path.join(instance.zookeeperDatadir,
"zookeeper_{}.db".format(abs(hash(cluster))))
charts = []
try:
sql = """
SELECT
ip,
{0},
avg(alive_clients),
avg(receive_packets),
avg(sent_packets),
avg(out_requests),
avg(watch_count),
avg(znode_count),
avg(data_size),
avg(open_file_count),
avg(min_latency),
avg(avg_latency),
avg(max_latency)
FROM `{1}`
WHERE {0} BETWEEN {2} AND {3}
GROUP BY {0}, ip
""".format(column, "zkstatus", start, end)
with SQLite3(db) as client:
data = yield client.read(sql)
meta = ("ip", "timestamp", "aliveClient", "recvPkt", "sentPkt",
"outRest", "watchCount", "znodeCount", "dataSize",
"openFileCount", "minLatency", "avgLatency", "maxLatency")
data = [dict(zip(meta, x)) for x in data]
for k in meta:
if k not in ["ip", "timestamp"]:
chart, items = [], {}
for metric in data:
ip = metric["ip"]
if not items.get(ip):
items[ip] = []
items[ip].append(
[metric["timestamp"] * 1000,
int(metric[k])])
for k, v in items.iteritems():
chart.append({"name": k, "data": v})
charts.append(chart)
except Exception:
pass
raise Return(charts)
def getHighcharts(self,
db,
column,
start,
end,
category,
table,
partitions,
group=None,
delta=False):
""" 返回Hightchats数据 """
chart, items = [], {}
try:
if group:
sql = """
SELECT {0}, {1}
FROM `{2}`
WHERE category = '{3}'
AND group_name = '%s'
AND {0} BETWEEN {4} AND {5}
GROUP BY {0}
""" % group
else:
sql = """
SELECT {0}, {1}
FROM `{2}`
WHERE category = '{3}'
AND group_name = (
SELECT group_name
FROM `{2}`
ORDER BY id DESC
LIMIT 1
)
AND {0} BETWEEN {4} AND {5}
GROUP BY {0}
"""
with SQLite3(db) as client:
data = yield client.read(
sql.format(
column,
",".join(["avg(p{0})".format(p) for p in partitions]),
table, category, start, end))
meta = "timestamp, {}".format(",".join(
["{0}".format(p) for p in partitions]))
meta = tuple([x.strip() for x in meta.split(",")])
data = [dict(zip(meta, x)) for x in data]
for metric in data:
for k, v in metric.iteritems():
if k != "timestamp":
if not items.get(k):
items[k] = []
items[k].append([metric["timestamp"] * 1000, int(v)])
for name, data in items.iteritems():
chart.append({"name": "%02s" % name, "data": data})
if delta:
# 计算highchart图表delta数据
metric = chart
chart = []
for item in metric:
data = item["data"]
index = range(len(data))
result = []
for i in index:
if i + 1 <= index[-1]:
result.append(
[data[i + 1][0], data[i + 1][1] - data[i][1]])
chart.append({"name": item["name"], "data": result})
chart = sorted(chart, key=lambda x: x["name"])
except Exception:
pass
raise Return(chart)