def plot_all_phases(db_name, host_ip, bucket_name):
# initialize seriesly client
db = Seriesly()[db_name]
db_event = Seriesly()['event']
# plot all metrics to PNG images
outdir = mkdtemp()
# get a set of all unique keys based on time range
all_docs = db.get_all()
all_keys = set(key for doc in all_docs.itervalues()
for key in doc.iterkeys())
# get system test phase info and plot phase by phase
all_event_docs = db_event.get_all()
phases_info = {}
for doc in all_event_docs.itervalues():
phases_info[int(doc.keys()[0])] = doc.values()[0]
phases_info.keys().sort()
phases = []
for v in phases_info.itervalues():
phases.append(v)
num_phases = len(phases)
run_id = ''
for i in range(num_phases):
if i == 0:
run_id = phases[i]['run_id']
start_time = phases[i].values()[0]
start_time = int(start_time[:10])
end_time = 0
if i == num_phases-1:
end_time = str(time.time())
end_time = int(end_time[:10])
else:
end_time = phases[i+1].values()[0]
end_time = int(end_time[:10])
for metric in all_keys:
#print metric
if '/' not in metric: # views and xdcr stuff
query = get_query(metric, host_ip, bucket_name, start_time, end_time)
if len(query) > 0:
plot_metric(db, metric, query, outdir, i, phases[i].keys()[0])
# try:
# subprocess.call(['convert', '{0}/*'.format(outdir), 'report.pdf'])
# print "PDF report was successfully generated!"
# except OSError:
plot_metric_single_value("average", outdir, num_phases)
plot_metric_single_value("90th", outdir, num_phases)
plot_metric_single_value("absolute_time", outdir, num_phases)
print "All images saved to: {0}".format(outdir)
return outdir, run_id
def plot_all_phases(db_name, host_ip, bucket_name):
# initialize seriesly client
db = Seriesly()[db_name]
db_event = Seriesly()['event']
# plot all metrics to PNG images
outdir = mkdtemp()
# get a set of all unique keys based on time range
all_docs = db.get_all()
all_keys = set(key for doc in all_docs.itervalues()
for key in doc.iterkeys())
# get system test phase info and plot phase by phase
all_event_docs = db_event.get_all()
phases_info = {}
for doc in all_event_docs.itervalues():
phases_info[int(doc.keys()[0])] = doc.values()[0]
phases_info.keys().sort()
phases = []
for v in phases_info.itervalues():
phases.append(v)
num_phases = len(phases)
run_id = ''
for i in range(num_phases)[1:]:
if i == 1:
run_id = phases[i]['run_id']
start_time = phases[i].values()[0]
start_time = int(start_time[:10])
end_time = 0
if i == num_phases-1:
end_time = str(time.time())
end_time = int(end_time[:10])
else:
end_time = phases[i+1].values()[0]
end_time = int(end_time[:10])
for metric in all_keys:
#print metric
if '/' not in metric: # views and xdcr stuff
query = get_query(metric, host_ip, bucket_name, start_time, end_time)
if len(query) > 0:
plot_metric(db, metric, query, outdir, i, phases[i].keys()[0])
for key in TABLE.keys():
plot_metric_single_value(key, outdir, num_phases)
print "All images saved to: {0}".format(outdir)
return outdir, run_id
def main():
# parse database name from cli arguments
db_name = parse_args()
# initialize seriesly client
db = Seriesly()[db_name]
# get a set of all unique keys
all_docs = db.get_all()
all_keys = set(key for doc in all_docs.itervalues()
for key in doc.iterkeys())
# plot all metrics to PNG images
outdir = mkdtemp()
for metric in all_keys:
print metric
if '/' not in metric: # views and xdcr stuff
keys, values = get_metric(db, metric)
plot_metric(metric, keys, values, outdir)
try:
subprocess.call(['convert', '{0}/*'.format(outdir), 'report.pdf'])
print "PDF report was successfully generated!"
except OSError:
print "All images saved to: {0}".format(outdir)
def generate_index_file(storage_folder, test_file):
db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event']
all_event_docs = db_event.get_all()
phases_info = {}
for doc in all_event_docs.itervalues():
phases_info[int(doc.keys()[0])] = doc.values()[0]
phases_info.keys().sort()
num_phases = len(phases_info.keys())
run_id = phases_info[1]['desc']
run_id = run_id.replace(" ", "_")
run_id = run_id.replace(",", "_")
content = ""
json_data = open(test_file)
tests = json.load(json_data)
for i in range(num_phases)[1:]:
sub_folder = storage_folder + "phase" + str(i) + "/"
content += "<a style=\"font-family:arial;color:black;font-size:20px;\" href=\"%s\">%s</a><p>" % ("phase" + str(i) , "phase" + str(i))
if str(i) in tests["phases"]:
content += json.dumps(tests["phases"][str(i)], indent=10, sort_keys=True) + "<p>"
files = [ f for f in os.listdir(sub_folder) if os.path.isfile(os.path.join(sub_folder, f))]
for f in files:
content += "<a href=\"%s\"> %s</a><p>" % ("phase" + str(i) + "/" + f, f)
html_path = storage_folder + "index.html"
file1 = open(html_path, 'w')
file1.write(index_html % content)
def get_run_info(desc):
db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event']
all_event_docs = db_event.get_all()
phases_info = {}
for doc in all_event_docs.itervalues():
phases_info[int(doc.keys()[0])] = doc.values()[0]
phases_info.keys().sort()
run_info = ''
#will take the first name/desc value; but we assume that their values are the same for all phases
if desc == 'name':
for phase in phases_info:
if 'name' in phases_info[phase]:
run_info = phases_info[phase]['name']
break
if desc == 'build':
for phase in phases_info:
if 'desc' in phases_info[phase]:
run_info = phases_info[phase]['desc']
break
run_info = run_info.replace(" ", "_")
run_info = run_info.replace(",", "_")
return run_info
def generate_index_file(storage_folder, test_file):
db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event']
all_event_docs = db_event.get_all()
phases_info = {}
for doc in all_event_docs.itervalues():
phases_info[int(doc.keys()[0])] = doc.values()[0]
phases_info.keys().sort()
num_phases = len(phases_info.keys())
run_id = phases_info[1]['desc']
run_id = run_id.replace(" ", "_")
run_id = run_id.replace(",", "_")
content = ""
json_data = open(test_file)
tests = json.load(json_data)
for i in range(num_phases)[1:]:
sub_folder = storage_folder + "phase" + str(i) + "/"
content += "<a style=\"font-family:arial;color:black;font-size:20px;\" href=\"%s\">%s</a><p>" % (
"phase" + str(i), "phase" + str(i))
if str(i) in tests["phases"]:
content += json.dumps(
tests["phases"][str(i)], indent=10, sort_keys=True) + "<p>"
files = [
f for f in os.listdir(sub_folder)
if os.path.isfile(os.path.join(sub_folder, f))
]
for f in files:
content += "<a href=\"%s\"> %s</a><p>" % (
"phase" + str(i) + "/" + f, f)
html_path = storage_folder + "index.html"
file1 = open(html_path, 'w')
file1.write(index_html % content)
def get_run_info(desc):
db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event']
all_event_docs = db_event.get_all()
phases_info = {}
for doc in all_event_docs.itervalues():
phases_info[int(doc.keys()[0])] = doc.values()[0]
phases_info.keys().sort()
run_info = ''
#will take the first name/desc value; but we assume that their values are the same for all phases
if desc == 'name':
for phase in phases_info:
if 'name' in phases_info[phase]:
run_info = phases_info[phase]['name']
break
if desc == 'build':
for phase in phases_info:
if 'desc' in phases_info[phase]:
run_info = phases_info[phase]['desc']
break
run_info = run_info.replace(" ", "_")
run_info = run_info.replace(",", "_")
return run_info
def main(db_name):
# parse database name from cli arguments
#db_name = parse_args()
# initialize seriesly client
db = Seriesly()[db_name]
# get a set of all unique keys
all_docs = db.get_all()
all_keys = set(key for doc in all_docs.itervalues()
for key in doc.iterkeys())
# plot all metrics to PNG images
outdir = mkdtemp()
for metric in all_keys:
print metric
if '/' not in metric: # views and xdcr stuff
keys, values = get_metric(db, metric)
plot_metric(metric, keys, values, outdir)
try:
subprocess.call(['convert', '{0}/*'.format(outdir), 'report.pdf'])
print "PDF report was successfully generated!"
except OSError:
print "All images saved to: {0}".format(outdir)
return outdir
def plot_all_phases(cluster_name, buckets):
db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event']
# Get system test phase info and plot phase by phase
all_event_docs = db_event.get_all()
phases_info = {}
for doc in all_event_docs.values():
phases_info[int(list(doc.keys())[0])] = list(doc.values())[0]
list(phases_info.keys()).sort()
num_phases = len(list(phases_info.keys()))
run_id = store_report.get_run_info('name')
if not os.path.exists("%s" % run_id):
os.makedirs("%s" % run_id)
else:
shutil.rmtree("%s" % run_id)
os.makedirs("%s" % run_id)
for i in list(phases_info.keys()):
start_time = phases_info[i][[
name for name in list(phases_info[i].keys())
if (name != 'name' and name != 'desc')
][0]]
start_time = int(start_time[:10])
end_time = 0
if i == list(phases_info.keys())[-1]:
end_time = str(time.time())
end_time = int(end_time[:10])
else:
end_time = phases_info[i + 1][[
name for name in list(phases_info[i + 1].keys())
if (name != 'name' and name != 'desc')
][0]]
end_time = int(end_time[:10])
start_time_snapshot = datetime.datetime.fromtimestamp(
start_time).strftime('%m/%d/%Y %H:%M')
end_time_snapshot = datetime.datetime.fromtimestamp(end_time).strftime(
'%m/%d/%Y %H:%M')
snapshot_name = "phase-%d-%s" % (i, [
name for name in list(phases_info[i].keys())
if (name != 'name' and name != 'desc')
][0])
plot_use_cbmonitor(snapshot_name, cluster_name, start_time_snapshot,
end_time_snapshot)
prepare_folder_report(run_id, i)
store_90th_avg_value(buckets, start_time, end_time, run_id, i)
storage_folder = os.getcwd() + "/" + run_id + "/"
print("data stored in %s" % (storage_folder))
return storage_folder
def plot_all_phases(cluster_name, buckets):
db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event']
# Get system test phase info and plot phase by phase
all_event_docs = db_event.get_all()
phases_info = {}
for doc in all_event_docs.itervalues():
phases_info[int(doc.keys()[0])] = doc.values()[0]
phases_info.keys().sort()
num_phases = len(phases_info.keys())
run_id = store_report.get_run_info('name')
if not os.path.exists("%s" % run_id):
os.makedirs("%s" % run_id)
else:
shutil.rmtree("%s" % run_id)
os.makedirs("%s" % run_id)
for i in phases_info.keys():
start_time = phases_info[i][[name for name in phases_info[i].keys() if (name != 'name' and name != 'desc')][0]]
start_time = int(start_time[:10])
end_time = 0
if i == phases_info.keys()[-1]:
end_time = str(time.time())
end_time = int(end_time[:10])
else:
end_time = phases_info[i + 1][[name for name in phases_info[i + 1].keys() if (name != 'name' and name != 'desc')][0]]
end_time = int(end_time[:10])
start_time_snapshot = datetime.datetime.fromtimestamp(start_time).strftime('%m/%d/%Y %H:%M')
end_time_snapshot = datetime.datetime.fromtimestamp(end_time).strftime('%m/%d/%Y %H:%M')
snapshot_name = "phase-%d-%s" % (i, [name for name in phases_info[i].keys() if (name != 'name' and name != 'desc')][0])
plot_use_cbmonitor(snapshot_name, cluster_name, start_time_snapshot, end_time_snapshot)
prepare_folder_report(run_id, i)
store_90th_avg_value(buckets, start_time, end_time, run_id, i)
storage_folder = os.getcwd() + "/" + run_id + "/"
print "data stored in %s" % (storage_folder)
return storage_folder
from seriesly import Seriesly
db = Seriesly()["fast"]
doc = db.get_all()
time = {}
for k, v in doc.iteritems():
if "mc-curr_items" in v.keys() and "mc-host" in v.keys():
time[k] = {"item": v["mc-curr_items"], "ip": v["mc-host"]}
else:
time[k] = {"item": "No items"}
for k, v in sorted(time.iteritems()):
print k, v
