def plot_all_phases(cluster_name, buckets):
db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event']
# Get system test phase info and plot phase by phase
all_event_docs = db_event.get_all()
phases_info = {}
for doc in all_event_docs.values():
phases_info[int(list(doc.keys())[0])] = list(doc.values())[0]
list(phases_info.keys()).sort()
num_phases = len(list(phases_info.keys()))
run_id = store_report.get_run_info('name')
if not os.path.exists("%s" % run_id):
os.makedirs("%s" % run_id)
else:
shutil.rmtree("%s" % run_id)
os.makedirs("%s" % run_id)
for i in list(phases_info.keys()):
start_time = phases_info[i][[
name for name in list(phases_info[i].keys())
if (name != 'name' and name != 'desc')
][0]]
start_time = int(start_time[:10])
end_time = 0
if i == list(phases_info.keys())[-1]:
end_time = str(time.time())
end_time = int(end_time[:10])
else:
end_time = phases_info[i + 1][[
name for name in list(phases_info[i + 1].keys())
if (name != 'name' and name != 'desc')
][0]]
end_time = int(end_time[:10])
start_time_snapshot = datetime.datetime.fromtimestamp(
start_time).strftime('%m/%d/%Y %H:%M')
end_time_snapshot = datetime.datetime.fromtimestamp(end_time).strftime(
'%m/%d/%Y %H:%M')
snapshot_name = "phase-%d-%s" % (i, [
name for name in list(phases_info[i].keys())
if (name != 'name' and name != 'desc')
][0])
plot_use_cbmonitor(snapshot_name, cluster_name, start_time_snapshot,
end_time_snapshot)
prepare_folder_report(run_id, i)
store_90th_avg_value(buckets, start_time, end_time, run_id, i)
storage_folder = os.getcwd() + "/" + run_id + "/"
print("data stored in %s" % (storage_folder))
return storage_folder
def plot_all_phases(cluster_name, buckets):
db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event']
# Get system test phase info and plot phase by phase
all_event_docs = db_event.get_all()
phases_info = {}
for doc in all_event_docs.itervalues():
phases_info[int(doc.keys()[0])] = doc.values()[0]
phases_info.keys().sort()
num_phases = len(phases_info.keys())
run_id = store_report.get_run_info('name')
if not os.path.exists("%s" % run_id):
os.makedirs("%s" % run_id)
else:
shutil.rmtree("%s" % run_id)
os.makedirs("%s" % run_id)
for i in phases_info.keys():
start_time = phases_info[i][[name for name in phases_info[i].keys() if (name != 'name' and name != 'desc')][0]]
start_time = int(start_time[:10])
end_time = 0
if i == phases_info.keys()[-1]:
end_time = str(time.time())
end_time = int(end_time[:10])
else:
end_time = phases_info[i + 1][[name for name in phases_info[i + 1].keys() if (name != 'name' and name != 'desc')][0]]
end_time = int(end_time[:10])
start_time_snapshot = datetime.datetime.fromtimestamp(start_time).strftime('%m/%d/%Y %H:%M')
end_time_snapshot = datetime.datetime.fromtimestamp(end_time).strftime('%m/%d/%Y %H:%M')
snapshot_name = "phase-%d-%s" % (i, [name for name in phases_info[i].keys() if (name != 'name' and name != 'desc')][0])
plot_use_cbmonitor(snapshot_name, cluster_name, start_time_snapshot, end_time_snapshot)
prepare_folder_report(run_id, i)
store_90th_avg_value(buckets, start_time, end_time, run_id, i)
storage_folder = os.getcwd() + "/" + run_id + "/"
print "data stored in %s" % (storage_folder)
return storage_folder
else:
port = '8091'
try:
serverInfo = {'ip':ip, 'username':cfg.COUCHBASE_USER, 'password':cfg.COUCHBASE_PWD, 'port':port}
rest = RestConnection(serverInfo)
buckets = [bucket.name for bucket in rest.get_buckets()]
break
except Exception, e:
print e
release_number = options.release_number
platform = options.platform
test_name = options.test_name
if test_name == None:
test_name = store_report.get_run_info('name')
test_build = options.version
if test_build == None:
test_build = store_report.get_run_info('build')
storage_folder = plotter.plot_all_phases(cluster_name, buckets)
if None in [options.release_to_compare, options.version_to_compare]:
if options.release_to_compare is None:
print "release_to_compare was not set"
if options.version_to_compare is None:
print "version_to_compare was not set"
print 'Comparison of versions to be skipped'
else:
release_to_compare = options.release_to_compare
version_to_compare = options.version_to_compare
'username': cfg.COUCHBASE_USER,
'password': cfg.COUCHBASE_PWD,
'port': port
}
rest = RestConnection(serverInfo)
buckets = [bucket.name for bucket in rest.get_buckets()]
break
except Exception, e:
print e
release_number = options.release_number
platform = options.platform
test_name = options.test_name
if test_name == None:
test_name = store_report.get_run_info('name')
test_build = options.version
if test_build == None:
test_build = store_report.get_run_info('build')
storage_folder = plotter.plot_all_phases(cluster_name, buckets)
if None in [options.release_to_compare, options.version_to_compare]:
if options.release_to_compare is None:
print "release_to_compare was not set"
if options.version_to_compare is None:
print "version_to_compare was not set"
print 'Comparison of versions to be skipped'
else:
release_to_compare = options.release_to_compare
version_to_compare = options.version_to_compare
def main():
options, args = parse_args()
cluster_name = options.cluster_name
if options.cluster_name is None:
cluster_name = cfg.CB_CLUSTER_TAG
buckets = options.buckets
if buckets is None:
for ip in cfg.CLUSTER_IPS:
if ":" in ip:
port = ip[ip.find(":") + 1:]
ip = ip[0:ip.find(":")]
else:
port = '8091'
try:
serverInfo = {
'ip': ip,
'username': cfg.COUCHBASE_USER,
'password': cfg.COUCHBASE_PWD,
'port': port
}
rest = RestConnection(serverInfo)
buckets = [bucket.name for bucket in rest.get_buckets()]
break
except Exception as e:
print(e)
release_number = options.release_number
platform = options.platform
test_name = options.test_name
if test_name == None:
test_name = store_report.get_run_info('name')
test_build = options.version
if test_build == None:
test_build = store_report.get_run_info('build')
storage_folder = plotter.plot_all_phases(cluster_name, buckets)
if None in [options.release_to_compare, options.version_to_compare]:
if options.release_to_compare is None:
print("release_to_compare was not set")
if options.version_to_compare is None:
print("version_to_compare was not set")
print('Comparison of versions to be skipped')
else:
release_to_compare = options.release_to_compare
version_to_compare = options.version_to_compare
folders = os.walk(storage_folder).next()[1]
for folder in folders:
sub_folder = os.path.join(storage_folder, folder)
url_folder = "system-test-results/%s/%s/%s/%s/%s" % (
release_to_compare, platform, test_name, version_to_compare,
folder)
if not os.path.exists(url_folder):
os.makedirs(url_folder)
files = [
f for f in os.listdir(sub_folder)
if os.path.isfile(os.path.join(sub_folder, f))
]
for f in files:
os.system("wget %s/%s/%s -O %s/%s" %
(CBFS_HOST, url_folder, f, url_folder, f))
compare_stats.compare_by_folders(
storage_folder, "system-test-results/%s/%s/%s/%s" %
(release_to_compare, platform, test_name, version_to_compare))
generate_index_file(storage_folder, options.json_test_file)
store_report.store_report_cbfs(release_number, platform)
